Add Redirect policy to forward all domain APIs

cmd/server/cadence/server.go

@@ -166,7 +166,7 @@ func (s *server) startService() common.Daemon {
 	}
 	params.PProfInitializer = svcCfg.PProf.NewInitializer(params.Logger)
 
-	params.DCRedirectionPolicy = s.cfg.DCRedirectionPolicy
+	params.ClusterRedirectionPolicy = s.cfg.ClusterGroupMetadata.ClusterRedirectionPolicy
 
 	params.MetricsClient = metrics.NewClient(params.MetricScope, service.GetMetricsServiceIdx(params.Name, params.Logger))
 

common/config/cluster.go

@@ -45,6 +45,8 @@ type (
 		MasterClusterName string `yaml:"masterClusterName"`
 		// CurrentClusterName is the name of the cluster of current deployment
 		CurrentClusterName string `yaml:"currentClusterName"`
+		// ClusterRedirectionPolicy contains the cluster redirection policy for global domains
+		ClusterRedirectionPolicy *ClusterRedirectionPolicy `yaml:"clusterRedirectionPolicy"`
 		// ClusterGroup contains information for each cluster within the replication group
 		// Key is the clusterName
 		ClusterGroup map[string]ClusterInformation `yaml:"clusterGroup"`
@@ -85,7 +87,7 @@ func (m *ClusterGroupMetadata) Validate() error {
 	if m == nil {
 		return errors.New("ClusterGroupMetadata cannot be empty")
 	}
-
+	
 	if !m.EnableGlobalDomain {
 		log.Println("[WARN] Local domain is now deprecated. Please update config to enable global domain(ClusterGroupMetadata->EnableGlobalDomain)." +
 			"Global domain of single cluster has zero overhead, but only advantages for future migration and fail over. Please check Cadence documentation for more details.")
@@ -146,7 +148,7 @@ func (m *ClusterGroupMetadata) Validate() error {
 // FillDefaults populates default values for unspecified fields
 func (m *ClusterGroupMetadata) FillDefaults() {
 	if m == nil {
-		return
+		log.Fatal("ClusterGroupMetadata cannot be empty")
 	}
 
 	// TODO: remove this after 0.23 and mention a breaking change in config.

common/config/config.go

@@ -49,8 +49,8 @@ type (
 		ClusterGroupMetadata *ClusterGroupMetadata `yaml:"clusterGroupMetadata"`
 		// Deprecated: please use ClusterGroupMetadata
 		ClusterMetadata *ClusterGroupMetadata `yaml:"clusterMetadata"`
-		// DCRedirectionPolicy contains the frontend datacenter redirection policy
-		DCRedirectionPolicy DCRedirectionPolicy `yaml:"dcRedirectionPolicy"`
+		// Deprecated: please use ClusterRedirectionPolicy under ClusterGroupMetadata
+		DCRedirectionPolicy *ClusterRedirectionPolicy `yaml:"dcRedirectionPolicy"`
 		// Services is a map of service name to service config items
 		Services map[string]Service `yaml:"services"`
 		// Kafka is the config for connecting to kafka
@@ -334,10 +334,44 @@ type (
 		Encoding string `yaml:"encoding"`
 	}
 
-	// DCRedirectionPolicy contains the frontend datacenter redirection policy
-	DCRedirectionPolicy struct {
+	// ClusterRedirectionPolicy contains the frontend datacenter redirection policy
+	// When using XDC (global domain) feature to failover a domain from one cluster to another one, client may call the passive cluster to start /signal workflows etc.
+	// To have a seamless failover experience, cluster should use this forwarding option to forward those APIs to the active cluster.
+	ClusterRedirectionPolicy struct {
+		// Support "noop", "selected-apis-forwarding" and "all-domain-apis-forwarding", default (when empty) is "noop"
+		//
+		// 1) "noop" will not do any forwarding.
+		//
+		// 2) "all-domain-apis-forwarding" will forward all domain specific APIs(worker and non worker) if the current active domain is
+		// the same as "allDomainApisForwardingTargetCluster"( or "allDomainApisForwardingTargetCluster" is empty), otherwise it fallbacks to "selected-apis-forwarding". 
+		//
+		// 3) "selected-apis-forwarding" will forward all non-worker APIs including
+		// 1. StartWorkflowExecution
+		// 2. SignalWithStartWorkflowExecution
+		// 3. SignalWorkflowExecution
+		// 4. RequestCancelWorkflowExecution
+		// 5. TerminateWorkflowExecution
+		// 6. QueryWorkflow
+		// 7. ResetWorkflow
+		//
+		// Both "selected-apis-forwarding" and "all-domain-apis-forwarding" can work with EnableDomainNotActiveAutoForwarding dynamicconfig to select certain domains using the policy.
+		//
+		// Usage recommendation: when enabling XDC(global domain) feature, either "all-domain-apis-forwarding" or "selected-apis-forwarding" should be used to ensure seamless domain failover(high availability)
+		// Depending on the cost of cross cluster calls :
+		//
+		// 1) If the network communication overhead is high(e.g., clusters are in remote datacenters of different region), then should use "selected-apis-forwarding".
+		// But you must ensure a different set of workers with the same workflow & activity code are connected to each Cadence cluster.
+		// 
+		// 2) If the network communication overhead is low (e.g. in the same datacenter, mostly for cluster migration usage), then you can use "all-domain-apis-forwarding". Then only one set of
+		// workflow & activity worker connected of one of the Cadence cluster is enough as all domain APIs are forwarded. See more details in documentation of cluster migration section.
+		// Usually "allDomainApisForwardingTargetCluster" should be empty(default value) except for very rare cases: you have more than two clusters and some are in a remote region but some are in local region.    
 		Policy string `yaml:"policy"`
-		ToDC   string `yaml:"toDC"`
+		// A supplement for "all-domain-apis-forwarding" policy. It decides how the policy fallback to  "selected-apis-forwarding" policy.
+		// If this is not empty, and current domain is not active in the value of allDomainApisForwardingTargetCluster, then the policy will fallback to "selected-apis-forwarding" policy.
+		// Default is empty, meaning that all requests will not fallback.
+		AllDomainApisForwardingTargetCluster string `yaml:"allDomainApisForwardingTargetCluster"`
+		// Not being used, but we have to keep it so that config loading is not broken
+		ToDC string `yaml:"toDC"`
 	}
 
 	// Metrics contains the config items for metrics subsystem
@@ -527,6 +561,11 @@ func (c *Config) fillDefaults() {
 			c.PublicClient.HostPort = fmt.Sprintf("%v:%v", host, thriftPort)
 		}
 	}
+
+	if c.ClusterGroupMetadata.ClusterRedirectionPolicy == nil && c.DCRedirectionPolicy != nil {
+		log.Println("[WARN] dcRedirectionPolicy config is deprecated. Please replace it with clusterRedirectionPolicy.")
+		c.ClusterGroupMetadata.ClusterRedirectionPolicy = c.DCRedirectionPolicy
+	}
 }
 
 // String converts the config object into a string

common/config/config_test.go

@@ -211,6 +211,7 @@ func TestConfigErrorInAuthorizationConfig(t *testing.T) {
 				Enable: true,
 			},
 		},
+		ClusterGroupMetadata: &ClusterGroupMetadata{},
 	}
 
 	err := cfg.ValidateAndFillDefaults()

common/dynamicconfig/constants.go

@@ -122,10 +122,12 @@ const (
 	// Default value: the value in static config: common.Config.Archival.Visibility.EnableRead
 	// Allowed filters: N/A
 	EnableReadFromVisibilityArchival
-	// EnableDomainNotActiveAutoForwarding is whether enabling DC auto forwarding to active cluster for signal / start / signal with start API if domain is not active
+	// EnableDomainNotActiveAutoForwarding decides requests form which domain will be forwarded to active cluster if domain is not active in current cluster.
+	// Only when "selected-api-forwarding" or "all-domain-apis-forwarding" is the policy in ClusterRedirectionPolicy(in static config).
+	// If the policy is "noop"(default) this flag is not doing anything.
 	// KeyName: system.enableDomainNotActiveAutoForwarding
 	// Value type: Bool
-	// Default value: TRUE
+	// Default value: TRUE (meaning all domains)
 	// Allowed filters: DomainName
 	EnableDomainNotActiveAutoForwarding
 	// EnableGracefulFailover is whether enabling graceful failover

common/resource/params.go

@@ -48,25 +48,25 @@ type (
 		Logger          log.Logger
 		ThrottledLogger log.Logger
 
-		MetricScope         tally.Scope
-		MembershipFactory   MembershipMonitorFactory
-		RPCFactory          common.RPCFactory
-		PProfInitializer    common.PProfInitializer
-		PersistenceConfig   config.Persistence
-		ClusterMetadata     cluster.Metadata
-		ReplicatorConfig    config.Replicator
-		MetricsClient       metrics.Client
-		MessagingClient     messaging.Client
-		BlobstoreClient     blobstore.Client
-		ESClient            es.GenericClient
-		ESConfig            *config.ElasticSearchConfig
-		DynamicConfig       dynamicconfig.Client
-		DCRedirectionPolicy config.DCRedirectionPolicy
-		PublicClient        workflowserviceclient.Interface
-		ArchivalMetadata    archiver.ArchivalMetadata
-		ArchiverProvider    provider.ArchiverProvider
-		Authorizer          authorization.Authorizer // NOTE: this can be nil. If nil, AccessControlledHandlerImpl will initiate one with config.Authorization
-		AuthorizationConfig config.Authorization     // NOTE: empty(default) struct will get a authorization.NoopAuthorizer
+		MetricScope              tally.Scope
+		MembershipFactory        MembershipMonitorFactory
+		RPCFactory               common.RPCFactory
+		PProfInitializer         common.PProfInitializer
+		PersistenceConfig        config.Persistence
+		ClusterMetadata          cluster.Metadata
+		ReplicatorConfig         config.Replicator
+		MetricsClient            metrics.Client
+		MessagingClient          messaging.Client
+		BlobstoreClient          blobstore.Client
+		ESClient                 es.GenericClient
+		ESConfig                 *config.ElasticSearchConfig
+		DynamicConfig            dynamicconfig.Client
+		ClusterRedirectionPolicy *config.ClusterRedirectionPolicy
+		PublicClient             workflowserviceclient.Interface
+		ArchivalMetadata         archiver.ArchivalMetadata
+		ArchiverProvider         provider.ArchiverProvider
+		Authorizer               authorization.Authorizer // NOTE: this can be nil. If nil, AccessControlledHandlerImpl will initiate one with config.Authorization
+		AuthorizationConfig      config.Authorization     // NOTE: empty(default) struct will get a authorization.NoopAuthorizer
 	}
 
 	// MembershipMonitorFactory provides a bootstrapped membership monitor

common/rpc.go

@@ -47,10 +47,6 @@ const (
 	// ClientImplHeaderName refers to the name of the
 	// header that contains the client implementation
 	ClientImplHeaderName = "cadence-client-name"
-	// EnforceDCRedirection refers to a boolean string of whether
-	// to enforce DCRedirection(auto-forwarding)
-	// Will be removed in the future: https://github.com/uber/cadence/issues/2304
-	EnforceDCRedirection = "cadence-enforce-dc-redirection"
 	// AuthorizationTokenHeaderName refers to the jwt token in the request
 	AuthorizationTokenHeaderName = "cadence-authorization"
 )

config/development_xdc_cluster0.yaml

@@ -73,6 +73,8 @@ clusterGroupMetadata:
   failoverVersionIncrement: 10
   primaryClusterName: "cluster0"
   currentClusterName: "cluster0"
+  clusterRedirectionPolicy:
+    policy: "all-domain-apis-forwarding" # if network communication overhead between clusters is high, consider use "selected-apis-forwarding" instead, but workflow/activity workers need to be connected to each cluster to keep high availability
   clusterGroup:
     cluster0:
       enabled: true
@@ -93,9 +95,7 @@ clusterGroupMetadata:
       rpcAddress: "localhost:9833" # this is to let worker service and XDC replicator connected to the frontend service. In cluster setup, localhost will not work
       rpcTransport: "grpc"
 
-dcRedirectionPolicy:
-  policy: "selected-apis-forwarding"
-  toDC: ""
+
 
 archival:
   history:

config/development_xdc_cluster1.yaml

@@ -73,6 +73,8 @@ clusterGroupMetadata:
   failoverVersionIncrement: 10
   primaryClusterName: "cluster0"
   currentClusterName: "cluster1"
+  clusterRedirectionPolicy:
+    policy: "all-domain-apis-forwarding" # if network communication overhead between clusters is high, consider use "selected-apis-forwarding" instead, but workflow/activity workers need to be connected to each cluster to keep high availability
   clusterGroup:
     cluster0:
       enabled: true
@@ -93,10 +95,6 @@ clusterGroupMetadata:
       rpcAddress: "localhost:9833" # this is to let worker service and XDC replicator connected to the frontend service. In cluster setup, localhost will not work
       rpcTransport: "grpc"
 
-dcRedirectionPolicy:
-  policy: "selected-apis-forwarding"
-  toDC: ""
-
 archival:
   history:
     status: "enabled"

config/development_xdc_cluster2.yaml

@@ -73,6 +73,8 @@ clusterGroupMetadata:
   failoverVersionIncrement: 10
   primaryClusterName: "cluster0"
   currentClusterName: "cluster2"
+  clusterRedirectionPolicy:
+    policy: "all-domain-apis-forwarding" # if network communication overhead between clusters is high, consider use "selected-apis-forwarding" instead, but workflow/activity workers need to be connected to each cluster to keep high availability
   clusterGroup:
     cluster0:
       enabled: true
@@ -93,10 +95,6 @@ clusterGroupMetadata:
       rpcAddress: "localhost:9833" # this is to let worker service and XDC replicator connected to the frontend service. In cluster setup, localhost will not work
       rpcTransport: "grpc"
 
-dcRedirectionPolicy:
-  policy: "selected-apis-forwarding"
-  toDC: ""
-
 archival:
   history:
     status: "enabled"

host/onebox.go

@@ -389,7 +389,7 @@ func (c *cadenceImpl) GetHistoryClient() historyClient.Client {
 
 func (c *cadenceImpl) startFrontend(hosts map[string][]string, startWG *sync.WaitGroup) {
 	params := new(resource.Params)
-	params.DCRedirectionPolicy = config.DCRedirectionPolicy{}
+	params.ClusterRedirectionPolicy = &config.ClusterRedirectionPolicy{}
 	params.Name = service.Frontend
 	params.Logger = c.logger
 	params.ThrottledLogger = c.logger