diff --git a/common/isolationgroup/defaultisolationgroupstate/state.go b/common/isolationgroup/defaultisolationgroupstate/state.go index 4aa96548886..7faedab0e82 100644 --- a/common/isolationgroup/defaultisolationgroupstate/state.go +++ b/common/isolationgroup/defaultisolationgroupstate/state.go @@ -74,14 +74,12 @@ func NewDefaultIsolationGroupStateWatcherWithConfigStoreClient( }, nil } -func (z *defaultIsolationGroupStateHandler) AvailableIsolationGroupsByDomainID(ctx context.Context, domainID string, tasklistName string, availablePollerIsolationGroups []string) (types.IsolationGroupConfiguration, error) { +func (z *defaultIsolationGroupStateHandler) AvailableIsolationGroupsByDomainID(ctx context.Context, domainID string, _ string) (types.IsolationGroupConfiguration, error) { state, err := z.getByDomainID(ctx, domainID) if err != nil { return nil, fmt.Errorf("unable to get isolation group state: %w", err) } - availableIsolationGroupsCfg := isolationGroupHealthyListToConfig(availablePollerIsolationGroups) - scope := z.createAvailableisolationGroupMetricsScope(domainID, tasklistName) - return availableIG(z.config.AllIsolationGroups(), availableIsolationGroupsCfg, state.Global, state.Domain, scope), nil + return availableIG(z.config.AllIsolationGroups(), state.Global, state.Domain), nil } func (z *defaultIsolationGroupStateHandler) IsDrained(ctx context.Context, domain string, isolationGroup string) (bool, error) { @@ -162,44 +160,34 @@ func (z *defaultIsolationGroupStateHandler) get(ctx context.Context, domain stri return ig, nil } -func (z *defaultIsolationGroupStateHandler) createAvailableisolationGroupMetricsScope(domainID string, tasklistName string) metrics.Scope { - domainName, _ := z.domainCache.GetDomainName(domainID) - return z.metricsClient.Scope(metrics.GetAvailableIsolationGroupsScope). - Tagged(metrics.DomainTag(domainName)). - Tagged(metrics.TaskListTag(tasklistName)) -} - // A simple explicit deny-based isolation group implementation func availableIG( allIsolationGroups []string, - availablePollers types.IsolationGroupConfiguration, global types.IsolationGroupConfiguration, domain types.IsolationGroupConfiguration, - scope metrics.Scope, ) types.IsolationGroupConfiguration { out := types.IsolationGroupConfiguration{} for _, isolationGroup := range allIsolationGroups { - _, hasAvailablePollers := availablePollers[isolationGroup] globalCfg, hasGlobalConfig := global[isolationGroup] domainCfg, hasDomainConfig := domain[isolationGroup] if hasGlobalConfig { if globalCfg.State == types.IsolationGroupStateDrained { - scope.Tagged(metrics.PollerIsolationGroupTag(isolationGroup)).IncCounter(metrics.IsolationGroupStateDrained) + out[isolationGroup] = types.IsolationGroupPartition{ + Name: isolationGroup, + State: types.IsolationGroupStateDrained, + } continue } } if hasDomainConfig { if domainCfg.State == types.IsolationGroupStateDrained { - scope.Tagged(metrics.PollerIsolationGroupTag(isolationGroup)).IncCounter(metrics.IsolationGroupStateDrained) + out[isolationGroup] = types.IsolationGroupPartition{ + Name: isolationGroup, + State: types.IsolationGroupStateDrained, + } continue } } - if !hasAvailablePollers { - // we don't attempt to dispatch tasks to isolation groups where there are no pollers - scope.Tagged(metrics.PollerIsolationGroupTag(isolationGroup)).IncCounter(metrics.IsolationGroupStatePollerUnavailable) - continue - } - scope.Tagged(metrics.PollerIsolationGroupTag(isolationGroup)).IncCounter(metrics.IsolationGroupStateHealthy) out[isolationGroup] = types.IsolationGroupPartition{ Name: isolationGroup, State: types.IsolationGroupStateHealthy, @@ -223,14 +211,3 @@ func isDrained(isolationGroup string, global types.IsolationGroupConfiguration, } return false } - -func isolationGroupHealthyListToConfig(igs []string) types.IsolationGroupConfiguration { - out := make(types.IsolationGroupConfiguration, len(igs)) - for _, ig := range igs { - out[ig] = types.IsolationGroupPartition{ - Name: ig, - State: types.IsolationGroupStateHealthy, - } - } - return out -} diff --git a/common/isolationgroup/defaultisolationgroupstate/state_test.go b/common/isolationgroup/defaultisolationgroupstate/state_test.go index 5583a39593b..1087eebfe9f 100644 --- a/common/isolationgroup/defaultisolationgroupstate/state_test.go +++ b/common/isolationgroup/defaultisolationgroupstate/state_test.go @@ -74,24 +74,21 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { json.Unmarshal(validCfgDataDrained, &dynamicConfigResponseDrained) tests := map[string]struct { - availablePollerIsolationGroups []string - dcAffordance func(client *dynamicconfig.MockClient) - domainAffordance func(mock *cache.MockDomainCache) - cfg defaultConfig - expected types.IsolationGroupConfiguration - expectedErr error + dcAffordance func(client *dynamicconfig.MockClient) + domainAffordance func(mock *cache.MockDomainCache) + cfg defaultConfig + expected types.IsolationGroupConfiguration + expectedErr error }{ "normal case - feature is disabled": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return false }, - AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, + AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2"} }, }, dcAffordance: func(client *dynamicconfig.MockClient) {}, domainAffordance: func(mock *cache.MockDomainCache) { domainResponse := cache.NewDomainCacheEntryForTest(&persistence.DomainInfo{ID: "domain-id", Name: "domain"}, &persistence.DomainConfig{}, true, nil, 0, nil, 0, 0, 0) mock.EXPECT().GetDomainByID("domain-id").Return(domainResponse, nil) - mock.EXPECT().GetDomainName("domain-id").Return("domain", nil) }, expected: types.IsolationGroupConfiguration{ "zone-1": { @@ -106,10 +103,9 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { }, "normal case - no drains present - no configuration specifying a drain - feature is enabled": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, - AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, + AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2"} }, }, dcAffordance: func(client *dynamicconfig.MockClient) { client.EXPECT().GetListValue( @@ -120,7 +116,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { domainAffordance: func(mock *cache.MockDomainCache) { domainResponse := cache.NewDomainCacheEntryForTest(&persistence.DomainInfo{ID: "domain-id", Name: "domain"}, &persistence.DomainConfig{}, true, nil, 0, nil, 0, 0, 0) mock.EXPECT().GetDomainByID("domain-id").Return(domainResponse, nil) - mock.EXPECT().GetDomainName("domain-id").Return("domain", nil) mock.EXPECT().GetDomain("domain").Return(domainResponse, nil).AnyTimes() }, expected: types.IsolationGroupConfiguration{ @@ -136,10 +131,9 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { }, "normal case - one drain present - no configuration specifying a drain - feature is enabled": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, - AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, + AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2"} }, }, dcAffordance: func(client *dynamicconfig.MockClient) { client.EXPECT().GetListValue( @@ -150,10 +144,13 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { domainAffordance: func(mock *cache.MockDomainCache) { domainResponse := cache.NewDomainCacheEntryForTest(&persistence.DomainInfo{ID: "domain-id", Name: "domain"}, &persistence.DomainConfig{}, true, nil, 0, nil, 0, 0, 0) mock.EXPECT().GetDomainByID("domain-id").Return(domainResponse, nil) - mock.EXPECT().GetDomainName("domain-id").Return("domain", nil) mock.EXPECT().GetDomain("domain").Return(domainResponse, nil).AnyTimes() }, expected: types.IsolationGroupConfiguration{ + "zone-1": { + Name: "zone-1", + State: types.IsolationGroupStateDrained, + }, "zone-2": { Name: "zone-2", State: types.IsolationGroupStateHealthy, @@ -161,10 +158,9 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { }, }, "expected case - no global drain data configured": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, - AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, + AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2"} }, }, dcAffordance: func(client *dynamicconfig.MockClient) { client.EXPECT().GetListValue( @@ -175,7 +171,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { domainAffordance: func(mock *cache.MockDomainCache) { domainResponse := cache.NewDomainCacheEntryForTest(&persistence.DomainInfo{ID: "domain-id", Name: "domain"}, &persistence.DomainConfig{}, true, nil, 0, nil, 0, 0, 0) mock.EXPECT().GetDomainByID("domain-id").Return(domainResponse, nil) - mock.EXPECT().GetDomainName("domain-id").Return("domain", nil) mock.EXPECT().GetDomain("domain").Return(domainResponse, nil).AnyTimes() }, expected: types.IsolationGroupConfiguration{ @@ -190,7 +185,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { }, }, "pathological case - problems with global drain data - 1": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, @@ -210,7 +204,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { expectedErr: errors.New("unable to get isolation group state: could not resolve global drains in an error"), }, "pathological case - problems with domain drain data - cannot resolve domain": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, @@ -224,7 +217,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { }, "pathological case - problems with global drain data - malformed data returned 1": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, @@ -244,7 +236,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { expectedErr: errors.New("unable to get isolation group state: could not resolve global drains in an error"), }, "pathological case - problems with domain drain data - malformed data returned 1": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, @@ -260,7 +251,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { expectedErr: errors.New("unable to get isolation group state: could not resolve domain in isolationGroup handler: a failure"), }, "pathological case - problems with domain drain data - malformed data returned 2": { - availablePollerIsolationGroups: []string{"zone-1", "zone-2"}, cfg: defaultConfig{ IsolationGroupEnabled: func(string) bool { return true }, AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, @@ -275,26 +265,6 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { expected: nil, expectedErr: errors.New("unable to get isolation group state: could not resolve domain in isolationGroup handler: %!w()"), }, - "pathological case - no available pollers": { - availablePollerIsolationGroups: nil, - cfg: defaultConfig{ - IsolationGroupEnabled: func(string) bool { return true }, - AllIsolationGroups: func() []string { return []string{"zone-1", "zone-2", "zone-3"} }, - }, - dcAffordance: func(client *dynamicconfig.MockClient) { - client.EXPECT().GetListValue( - dynamicconfig.DefaultIsolationGroupConfigStoreManagerGlobalMapping, - gomock.Any(), - ).Return(nil, nil) - }, - domainAffordance: func(mock *cache.MockDomainCache) { - domainResponse := cache.NewDomainCacheEntryForTest(&persistence.DomainInfo{ID: "domain-id", Name: "domain"}, &persistence.DomainConfig{}, true, nil, 0, nil, 0, 0, 0) - mock.EXPECT().GetDomainByID("domain-id").Return(domainResponse, nil) - mock.EXPECT().GetDomainName("domain-id").Return("domain", nil) - mock.EXPECT().GetDomain("domain").Return(domainResponse, nil).AnyTimes() - }, - expected: types.IsolationGroupConfiguration{}, - }, } for name, td := range tests { @@ -311,7 +281,7 @@ func TestAvailableIsolationGroupsHandler(t *testing.T) { config: td.cfg, metricsClient: metrics.NewNoopMetricsClient(), } - res, err := handler.AvailableIsolationGroupsByDomainID(context.TODO(), "domain-id", "a-tasklist", td.availablePollerIsolationGroups) + res, err := handler.AvailableIsolationGroupsByDomainID(context.TODO(), "domain-id", "a-tasklist") assert.Equal(t, td.expected, res) if td.expectedErr != nil { assert.Equal(t, td.expectedErr.Error(), err.Error()) @@ -445,7 +415,7 @@ func TestAvailableIsolationGroups(t *testing.T) { }, } - isolationGroupsSetB := types.IsolationGroupConfiguration{ + isolationGroupsWithCDrained := types.IsolationGroupConfiguration{ igA: { Name: igA, State: types.IsolationGroupStateHealthy, @@ -454,9 +424,13 @@ func TestAvailableIsolationGroups(t *testing.T) { Name: igB, State: types.IsolationGroupStateHealthy, }, + igC: { + Name: igC, + State: types.IsolationGroupStateDrained, + }, } - isolationGroupsSetC := types.IsolationGroupConfiguration{ + drainedC := types.IsolationGroupConfiguration{ igC: { Name: igC, State: types.IsolationGroupStateDrained, @@ -464,50 +438,30 @@ func TestAvailableIsolationGroups(t *testing.T) { } tests := map[string]struct { - globalIGCfg types.IsolationGroupConfiguration - domainIGCfg types.IsolationGroupConfiguration - availablePollers types.IsolationGroupConfiguration - expected types.IsolationGroupConfiguration + globalIGCfg types.IsolationGroupConfiguration + domainIGCfg types.IsolationGroupConfiguration + expected types.IsolationGroupConfiguration }{ "default behaviour - no drains - everything should be healthy": { - globalIGCfg: types.IsolationGroupConfiguration{}, - domainIGCfg: types.IsolationGroupConfiguration{}, - availablePollers: isolationGroupsAllHealthy, - expected: isolationGroupsAllHealthy, - }, - "default behaviour - no drains - only one zone is healthy in terms of pollers, should only return that": { globalIGCfg: types.IsolationGroupConfiguration{}, domainIGCfg: types.IsolationGroupConfiguration{}, - availablePollers: types.IsolationGroupConfiguration{ - igC: types.IsolationGroupPartition{ - Name: igC, - State: types.IsolationGroupStateHealthy, - }, - }, - expected: types.IsolationGroupConfiguration{ - igC: types.IsolationGroupPartition{ - Name: igC, - State: types.IsolationGroupStateHealthy, - }, - }, + expected: isolationGroupsAllHealthy, }, - "default behaviour - one is drained - should return remaining 1/2": { - globalIGCfg: types.IsolationGroupConfiguration{}, - availablePollers: isolationGroupsAllHealthy, - domainIGCfg: isolationGroupsSetC, // C is drained - expected: isolationGroupsSetB, // A and B + "default behaviour - drained at domain level": { + globalIGCfg: types.IsolationGroupConfiguration{}, + domainIGCfg: drainedC, // C is drained + expected: isolationGroupsWithCDrained, // A and B }, - "default behaviour - one is drained - should return remaining 2/2": { - globalIGCfg: isolationGroupsSetC, // C is drained - availablePollers: isolationGroupsAllHealthy, - domainIGCfg: types.IsolationGroupConfiguration{}, - expected: isolationGroupsSetB, // A and B + "default behaviour - drained at global level": { + globalIGCfg: drainedC, // C is drained + domainIGCfg: types.IsolationGroupConfiguration{}, + expected: isolationGroupsWithCDrained, // A and B }, } for name, td := range tests { t.Run(name, func(t *testing.T) { - assert.Equal(t, td.expected, availableIG(all, td.availablePollers, td.globalIGCfg, td.domainIGCfg, metrics.NewNoopMetricsClient().Scope(0))) + assert.Equal(t, td.expected, availableIG(all, td.globalIGCfg, td.domainIGCfg)) }) } } diff --git a/common/isolationgroup/interface.go b/common/isolationgroup/interface.go index c6c2a40b038..a538bc8876e 100644 --- a/common/isolationgroup/interface.go +++ b/common/isolationgroup/interface.go @@ -43,5 +43,5 @@ type State interface { // AvailableIsolationGroupsByDomainID returns the available isolation zones for a domain. // Takes into account global and domain zones - AvailableIsolationGroupsByDomainID(ctx context.Context, domainID string, tasklistName string, availableIsolationGroups []string) (types.IsolationGroupConfiguration, error) + AvailableIsolationGroupsByDomainID(ctx context.Context, domainID string, tasklistName string) (types.IsolationGroupConfiguration, error) } diff --git a/common/isolationgroup/isolation_group_mock.go b/common/isolationgroup/isolation_group_mock.go index bbede23ab75..80e70d09ceb 100644 --- a/common/isolationgroup/isolation_group_mock.go +++ b/common/isolationgroup/isolation_group_mock.go @@ -59,18 +59,18 @@ func (m *MockState) EXPECT() *MockStateMockRecorder { } // AvailableIsolationGroupsByDomainID mocks base method. -func (m *MockState) AvailableIsolationGroupsByDomainID(ctx context.Context, domainID, tasklistName string, availableIsolationGroups []string) (types.IsolationGroupConfiguration, error) { +func (m *MockState) AvailableIsolationGroupsByDomainID(ctx context.Context, domainID, tasklistName string) (types.IsolationGroupConfiguration, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AvailableIsolationGroupsByDomainID", ctx, domainID, tasklistName, availableIsolationGroups) + ret := m.ctrl.Call(m, "AvailableIsolationGroupsByDomainID", ctx, domainID, tasklistName) ret0, _ := ret[0].(types.IsolationGroupConfiguration) ret1, _ := ret[1].(error) return ret0, ret1 } // AvailableIsolationGroupsByDomainID indicates an expected call of AvailableIsolationGroupsByDomainID. -func (mr *MockStateMockRecorder) AvailableIsolationGroupsByDomainID(ctx, domainID, tasklistName, availableIsolationGroups interface{}) *gomock.Call { +func (mr *MockStateMockRecorder) AvailableIsolationGroupsByDomainID(ctx, domainID, tasklistName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolationGroupsByDomainID", reflect.TypeOf((*MockState)(nil).AvailableIsolationGroupsByDomainID), ctx, domainID, tasklistName, availableIsolationGroups) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolationGroupsByDomainID", reflect.TypeOf((*MockState)(nil).AvailableIsolationGroupsByDomainID), ctx, domainID, tasklistName) } // IsDrained mocks base method. diff --git a/common/metrics/defs.go b/common/metrics/defs.go index e6827aab0c0..939671881d5 100644 --- a/common/metrics/defs.go +++ b/common/metrics/defs.go @@ -2247,9 +2247,6 @@ const ( ParentClosePolicyProcessorSuccess ParentClosePolicyProcessorFailures - IsolationGroupStatePollerUnavailable - IsolationGroupStateDrained - IsolationGroupStateHealthy ValidatedWorkflowCount HashringViewIdentifier @@ -2638,8 +2635,7 @@ const ( StandbyClusterTasksCompletedCounterPerTaskList StandbyClusterTasksNotStartedCounterPerTaskList StandbyClusterTasksCompletionFailurePerTaskList - TaskIsolationExpiredPerTaskList - TaskIsolationErrorPerTaskList + TaskIsolationLeakPerTaskList NumMatchingMetrics ) @@ -2959,13 +2955,10 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{ ParentClosePolicyProcessorSuccess: {metricName: "parent_close_policy_processor_requests", metricType: Counter}, ParentClosePolicyProcessorFailures: {metricName: "parent_close_policy_processor_errors", metricType: Counter}, - IsolationGroupStatePollerUnavailable: {metricName: "isolation_group_poller_unavailable", metricType: Counter}, - IsolationGroupStateDrained: {metricName: "isolation_group_drained", metricType: Counter}, - IsolationGroupStateHealthy: {metricName: "isolation_group_healthy", metricType: Counter}, - ValidatedWorkflowCount: {metricName: "task_validator_count", metricType: Counter}, - HashringViewIdentifier: {metricName: "hashring_view_identifier", metricType: Counter}, - DescribeWorkflowStatusError: {metricName: "describe_wf_error", metricType: Counter}, - DescribeWorkflowStatusCount: {metricName: "describe_wf_status", metricType: Counter}, + ValidatedWorkflowCount: {metricName: "task_validator_count", metricType: Counter}, + HashringViewIdentifier: {metricName: "hashring_view_identifier", metricType: Counter}, + DescribeWorkflowStatusError: {metricName: "describe_wf_error", metricType: Counter}, + DescribeWorkflowStatusCount: {metricName: "describe_wf_status", metricType: Counter}, AsyncRequestPayloadSize: {metricName: "async_request_payload_size_per_domain", metricRollupName: "async_request_payload_size", metricType: Timer}, @@ -3335,8 +3328,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{ StandbyClusterTasksCompletedCounterPerTaskList: {metricName: "standby_cluster_tasks_completed_per_tl", metricType: Counter}, StandbyClusterTasksNotStartedCounterPerTaskList: {metricName: "standby_cluster_tasks_not_started_per_tl", metricType: Counter}, StandbyClusterTasksCompletionFailurePerTaskList: {metricName: "standby_cluster_tasks_completion_failure_per_tl", metricType: Counter}, - TaskIsolationExpiredPerTaskList: {metricName: "task_isolation_expired_per_tl", metricRollupName: "task_isolation_expired"}, - TaskIsolationErrorPerTaskList: {metricName: "task_isolation_error_per_tl", metricRollupName: "task_isolation_error"}, + TaskIsolationLeakPerTaskList: {metricName: "task_isolation_leak_per_tl", metricRollupName: "task_isolation_leak"}, }, Worker: { ReplicatorMessages: {metricName: "replicator_messages"}, diff --git a/common/metrics/tags.go b/common/metrics/tags.go index f76a42ebf13..a8e7d8ff36d 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -66,6 +66,7 @@ const ( isolationEnabled = "isolation_enabled" isolationGroup = "isolation_group" originalIsolationGroup = "original_isolation_group" + leakCause = "leak_cause" topic = "topic" mode = "mode" @@ -319,7 +320,10 @@ func OriginalIsolationGroupTag(group string) Tag { func IsolationGroupTag(group string) Tag { return simpleMetric{key: isolationGroup, value: sanitizer.Value(group)} +} +func IsolationLeakCause(cause string) Tag { + return simpleMetric{key: leakCause, value: sanitizer.Value(cause)} } // IsolationEnabledTag returns whether isolation is enabled diff --git a/common/partition/default-partitioner.go b/common/partition/default-partitioner.go index e37081fe70f..cb40b6c0f87 100644 --- a/common/partition/default-partitioner.go +++ b/common/partition/default-partitioner.go @@ -26,10 +26,11 @@ import ( "context" "errors" "fmt" + "slices" "github.com/uber/cadence/common/isolationgroup" "github.com/uber/cadence/common/log" - "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/metrics" "github.com/uber/cadence/common/types" ) @@ -39,9 +40,13 @@ const ( WorkflowIDKey = "wf-id" ) -// ErrNoIsolationGroupsAvailable is returned when there are no available isolation-groups -// this usually indicates a misconfiguration -var ErrNoIsolationGroupsAvailable = errors.New("no isolation-groups are available") +var ( + IsolationLeakCauseError = metrics.IsolationLeakCause("error") + IsolationLeakCauseGroupUnknown = metrics.IsolationLeakCause("group_unknown") + IsolationLeakCauseGroupDrained = metrics.IsolationLeakCause("group_drained") + IsolationLeakCauseNoRecentPollers = metrics.IsolationLeakCause("no_recent_pollers") + IsolationLeakCauseExpired = metrics.IsolationLeakCause("expired") +) // ErrInvalidPartitionConfig is returned when the required partitioning configuration // is missing due to misconfiguration @@ -73,7 +78,7 @@ func NewDefaultPartitioner( } } -func (r *defaultPartitioner) GetIsolationGroupByDomainID(ctx context.Context, pollerInfo PollerInfo, wfPartitionData PartitionConfig) (string, error) { +func (r *defaultPartitioner) GetIsolationGroupByDomainID(ctx context.Context, scope metrics.Scope, pollerInfo PollerInfo, wfPartitionData PartitionConfig) (string, error) { if wfPartitionData == nil { return "", ErrInvalidPartitionConfig } @@ -82,17 +87,25 @@ func (r *defaultPartitioner) GetIsolationGroupByDomainID(ctx context.Context, po return "", ErrInvalidPartitionConfig } - available, err := r.isolationGroupState.AvailableIsolationGroupsByDomainID(ctx, pollerInfo.DomainID, pollerInfo.TasklistName, pollerInfo.AvailableIsolationGroups) + available, err := r.isolationGroupState.AvailableIsolationGroupsByDomainID(ctx, pollerInfo.DomainID, pollerInfo.TasklistName) if err != nil { return "", fmt.Errorf("failed to get available isolation groups: %w", err) } - - if len(available) == 0 { - return "", ErrNoIsolationGroupsAvailable + scope = scope.Tagged(metrics.IsolationGroupTag(wfPartition.WorkflowStartIsolationGroup)) + group, ok := available[wfPartition.WorkflowStartIsolationGroup] + if !ok { + scope.Tagged(IsolationLeakCauseGroupUnknown).IncCounter(metrics.TaskIsolationLeakPerTaskList) + return "", nil } - - ig := r.pickIsolationGroup(wfPartition, available, pollerInfo) - return ig, nil + if group.State != types.IsolationGroupStateHealthy { + scope.Tagged(IsolationLeakCauseGroupDrained).IncCounter(metrics.TaskIsolationLeakPerTaskList) + return "", nil + } + if !slices.Contains(pollerInfo.AvailableIsolationGroups, wfPartition.WorkflowStartIsolationGroup) { + scope.Tagged(IsolationLeakCauseNoRecentPollers).IncCounter(metrics.TaskIsolationLeakPerTaskList) + return "", nil + } + return wfPartition.WorkflowStartIsolationGroup, nil } func mapPartitionConfigToDefaultPartitionConfig(config PartitionConfig) defaultWorkflowPartitionConfig { @@ -103,19 +116,3 @@ func mapPartitionConfigToDefaultPartitionConfig(config PartitionConfig) defaultW WFID: wfID, } } - -// picks an isolation group to run in. if the workflow was started there, it'll attempt to pin it, unless there is an explicit -// drain. -func (r *defaultPartitioner) pickIsolationGroup(wfPartition defaultWorkflowPartitionConfig, available types.IsolationGroupConfiguration, pollerInfo PollerInfo) string { - _, isAvailable := available[wfPartition.WorkflowStartIsolationGroup] - if isAvailable { - return wfPartition.WorkflowStartIsolationGroup - } - r.log.Debug("isolation group falling back to any zone", - tag.IsolationGroup(wfPartition.WorkflowStartIsolationGroup), - tag.PollerGroupsConfiguration(available), - tag.WorkflowTaskListName(pollerInfo.TasklistName), - tag.WorkflowID(wfPartition.WFID), - ) - return "" -} diff --git a/common/partition/default-partitioner_test.go b/common/partition/default-partitioner_test.go index ceee9116314..a96d0221ce5 100644 --- a/common/partition/default-partitioner_test.go +++ b/common/partition/default-partitioner_test.go @@ -32,74 +32,10 @@ import ( "github.com/uber/cadence/common/isolationgroup" "github.com/uber/cadence/common/log/testlogger" + "github.com/uber/cadence/common/metrics" "github.com/uber/cadence/common/types" ) -func TestPickingAZone(t *testing.T) { - - igA := string("isolationGroupA") - igB := string("isolationGroupB") - igC := string("isolationGroupC") - - isolationGroupsAllHealthy := types.IsolationGroupConfiguration{ - igA: { - Name: igA, - State: types.IsolationGroupStateHealthy, - }, - igB: { - Name: igB, - State: types.IsolationGroupStateHealthy, - }, - igC: { - Name: igC, - State: types.IsolationGroupStateHealthy, - }, - } - - tests := map[string]struct { - availablePartitionGroups types.IsolationGroupConfiguration - wfPartitionCfg defaultWorkflowPartitionConfig - expected string - expectedErr error - }{ - "default behaviour - wf starting in a zone/isolationGroup should stay there if everything's healthy": { - availablePartitionGroups: isolationGroupsAllHealthy, - wfPartitionCfg: defaultWorkflowPartitionConfig{ - WorkflowStartIsolationGroup: igA, - WFID: "BDF3D8D9-5235-4CE8-BBDF-6A37589C9DC7", - }, - expected: igA, - }, - "default behaviour - wf starting in a zone/isolationGroup must run in an available zone only. If not in available list, return no zone": { - availablePartitionGroups: isolationGroupsAllHealthy, - wfPartitionCfg: defaultWorkflowPartitionConfig{ - WorkflowStartIsolationGroup: string("something-else"), - WFID: "BDF3D8D9-5235-4CE8-BBDF-6A37589C9DC7", - }, - expected: "", - }, - "... and it should be deterministic": { - availablePartitionGroups: isolationGroupsAllHealthy, - wfPartitionCfg: defaultWorkflowPartitionConfig{ - WorkflowStartIsolationGroup: string("something-else"), - WFID: "BDF3D8D9-5235-4CE8-BBDF-6A37589C9DC7", - }, - expected: "", - }, - } - - for name, td := range tests { - t.Run(name, func(t *testing.T) { - partitioner := defaultPartitioner{ - log: testlogger.New(t), - isolationGroupState: nil, - } - res := partitioner.pickIsolationGroup(td.wfPartitionCfg, td.availablePartitionGroups, PollerInfo{}) - assert.Equal(t, td.expected, res) - }) - } -} - func TestDefaultPartitioner_GetIsolationGroupByDomainID(t *testing.T) { domainID := "some-domain-id" @@ -111,10 +47,14 @@ func TestDefaultPartitioner_GetIsolationGroupByDomainID(t *testing.T) { }, "zone-3": { Name: "zone-3", + State: types.IsolationGroupStateDrained, + }, + "zone-4": { + Name: "zone-4", State: types.IsolationGroupStateHealthy, }, } - isolationGroups := []string{"zone-1", "zone-2", "zone-3"} + availablePollers := []string{"zone-1", "zone-2", "zone-3"} tests := map[string]struct { stateAffordance func(state *isolationgroup.MockState) @@ -123,40 +63,49 @@ func TestDefaultPartitioner_GetIsolationGroupByDomainID(t *testing.T) { expectedValue string expectedError error }{ - "happy path - zone is available - zone pinning": { + "zone is available - zone pinning": { partitionKeyPassedIn: PartitionConfig{ IsolationGroupKey: "zone-2", WorkflowIDKey: "wf-id", }, incomingContext: context.Background(), stateAffordance: func(state *isolationgroup.MockState) { - state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist, isolationGroups).Return(validIsolationGroup, nil) + state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist).Return(validIsolationGroup, nil) }, expectedValue: "zone-2", }, - "happy path - zone is not - zone fallback": { + "unknown zone - fallback to any": { partitionKeyPassedIn: PartitionConfig{ IsolationGroupKey: "zone-1", WorkflowIDKey: "wf-id", }, incomingContext: context.Background(), stateAffordance: func(state *isolationgroup.MockState) { - state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist, isolationGroups).Return(validIsolationGroup, nil) + state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist).Return(validIsolationGroup, nil) }, expectedValue: "", }, - "Error condition - No zones listed though the feature is enabled": { + "zone is drained - fallback to any": { partitionKeyPassedIn: PartitionConfig{ - IsolationGroupKey: "zone-1", + IsolationGroupKey: "zone-3", + WorkflowIDKey: "wf-id", + }, + incomingContext: context.Background(), + stateAffordance: func(state *isolationgroup.MockState) { + state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist).Return(validIsolationGroup, nil) + }, + expectedValue: "", + }, + "no pollers - fallback to any": { + partitionKeyPassedIn: PartitionConfig{ + IsolationGroupKey: "zone-4", WorkflowIDKey: "wf-id", }, incomingContext: context.Background(), stateAffordance: func(state *isolationgroup.MockState) { - state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist, isolationGroups).Return( - types.IsolationGroupConfiguration{}, nil) + state.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), domainID, sampleTasklist).Return(validIsolationGroup, nil) }, expectedValue: "", - expectedError: errors.New("no isolation-groups are available"), }, "Error condition - No isolation-group information passed in": { partitionKeyPassedIn: PartitionConfig{}, @@ -180,10 +129,10 @@ func TestDefaultPartitioner_GetIsolationGroupByDomainID(t *testing.T) { ig := isolationgroup.NewMockState(ctrl) td.stateAffordance(ig) partitioner := NewDefaultPartitioner(testlogger.New(t), ig) - res, err := partitioner.GetIsolationGroupByDomainID(td.incomingContext, PollerInfo{ + res, err := partitioner.GetIsolationGroupByDomainID(td.incomingContext, metrics.NoopScope(metrics.Matching), PollerInfo{ DomainID: domainID, TasklistName: sampleTasklist, - AvailableIsolationGroups: isolationGroups, + AvailableIsolationGroups: availablePollers, }, td.partitionKeyPassedIn) assert.Equal(t, td.expectedValue, res) diff --git a/common/partition/interface.go b/common/partition/interface.go index e91fe17c02b..38f9461b718 100644 --- a/common/partition/interface.go +++ b/common/partition/interface.go @@ -24,7 +24,11 @@ package partition //go:generate mockgen -package $GOPACKAGE -source $GOFILE -destination partitioning_mock.go -self_package github.com/uber/cadence/common/partition -import "context" +import ( + "context" + + "github.com/uber/cadence/common/metrics" +) // PollerInfo captures relevant information from the poller side type PollerInfo struct { @@ -39,5 +43,5 @@ type Partitioner interface { // GetIsolationGroupByDomainID gets where the task workflow should be executing. Largely used by Matching // when determining which isolationGroup to place the tasks in. // Implementations ought to return (nil, nil) for when the feature is not enabled. - GetIsolationGroupByDomainID(ctx context.Context, pollerinfo PollerInfo, partitionKey PartitionConfig) (string, error) + GetIsolationGroupByDomainID(ctx context.Context, scope metrics.Scope, pollerinfo PollerInfo, partitionKey PartitionConfig) (string, error) } diff --git a/common/partition/partitioning_mock.go b/common/partition/partitioning_mock.go index 232f6d5a73e..f0abf0c5284 100644 --- a/common/partition/partitioning_mock.go +++ b/common/partition/partitioning_mock.go @@ -31,6 +31,8 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + + metrics "github.com/uber/cadence/common/metrics" ) // MockPartitioner is a mock of Partitioner interface. @@ -57,16 +59,16 @@ func (m *MockPartitioner) EXPECT() *MockPartitionerMockRecorder { } // GetIsolationGroupByDomainID mocks base method. -func (m *MockPartitioner) GetIsolationGroupByDomainID(ctx context.Context, pollerinfo PollerInfo, partitionKey PartitionConfig) (string, error) { +func (m *MockPartitioner) GetIsolationGroupByDomainID(ctx context.Context, scope metrics.Scope, pollerinfo PollerInfo, partitionKey PartitionConfig) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetIsolationGroupByDomainID", ctx, pollerinfo, partitionKey) + ret := m.ctrl.Call(m, "GetIsolationGroupByDomainID", ctx, scope, pollerinfo, partitionKey) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetIsolationGroupByDomainID indicates an expected call of GetIsolationGroupByDomainID. -func (mr *MockPartitionerMockRecorder) GetIsolationGroupByDomainID(ctx, pollerinfo, partitionKey interface{}) *gomock.Call { +func (mr *MockPartitionerMockRecorder) GetIsolationGroupByDomainID(ctx, scope, pollerinfo, partitionKey interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIsolationGroupByDomainID", reflect.TypeOf((*MockPartitioner)(nil).GetIsolationGroupByDomainID), ctx, pollerinfo, partitionKey) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIsolationGroupByDomainID", reflect.TypeOf((*MockPartitioner)(nil).GetIsolationGroupByDomainID), ctx, scope, pollerinfo, partitionKey) } diff --git a/common/resource/resource_test_utils.go b/common/resource/resource_test_utils.go index c0c32ec6225..753b80f7b09 100644 --- a/common/resource/resource_test_utils.go +++ b/common/resource/resource_test_utils.go @@ -165,7 +165,7 @@ func NewTest( partitionMock := partition.NewMockPartitioner(controller) mockZone := "zone1" - partitionMock.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(mockZone, nil) + partitionMock.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(mockZone, nil) scope := tally.NewTestScope("test", nil) diff --git a/service/frontend/admin/handler_test.go b/service/frontend/admin/handler_test.go index 3252161a59f..b4d4952519f 100644 --- a/service/frontend/admin/handler_test.go +++ b/service/frontend/admin/handler_test.go @@ -52,7 +52,6 @@ import ( "github.com/uber/cadence/common/membership" "github.com/uber/cadence/common/metrics" "github.com/uber/cadence/common/mocks" - "github.com/uber/cadence/common/partition" "github.com/uber/cadence/common/persistence" "github.com/uber/cadence/common/resource" "github.com/uber/cadence/common/service" @@ -917,21 +916,6 @@ func Test_UpdateGlobalIsolationGroups(t *testing.T) { } } -func Test_IsolationGroupsNotEnabled(t *testing.T) { - handler := adminHandlerImpl{ - Resource: &resource.Test{ - Logger: testlogger.New(t), - MetricsClient: metrics.NewNoopMetricsClient(), - }, - isolationGroups: nil, // valid state, the isolation-groups feature is not available for all persistence types - } - - _, err := handler.GetGlobalIsolationGroups(context.Background(), &types.GetGlobalIsolationGroupsRequest{}) - assert.ErrorAs(t, err, &partition.ErrNoIsolationGroupsAvailable) - _, err = handler.UpdateGlobalIsolationGroups(context.Background(), &types.UpdateGlobalIsolationGroupsRequest{}) - assert.ErrorAs(t, err, &partition.ErrNoIsolationGroupsAvailable) -} - func Test_GetDomainIsolationGroups(t *testing.T) { validResponse := types.GetDomainIsolationGroupsResponse{ diff --git a/service/matching/tasklist/task_list_manager.go b/service/matching/tasklist/task_list_manager.go index 79138ec29a6..bff007ffb05 100644 --- a/service/matching/tasklist/task_list_manager.go +++ b/service/matching/tasklist/task_list_manager.go @@ -871,6 +871,7 @@ func (c *taskListManagerImpl) getIsolationGroupForTask(ctx context.Context, task pollerIsolationGroups := c.getPollerIsolationGroups() group, err := c.partitioner.GetIsolationGroupByDomainID(ctx, + c.scope, partition.PollerInfo{ DomainID: taskInfo.DomainID, TasklistName: c.taskListID.name, @@ -879,7 +880,7 @@ func (c *taskListManagerImpl) getIsolationGroupForTask(ctx context.Context, task if err != nil { // if we're unable to get the isolation group, log the error and fallback to no isolation c.logger.Error("Failed to get isolation group from partition library", tag.IsolationGroup(startedIsolationGroup), tag.WorkflowID(taskInfo.WorkflowID), tag.WorkflowRunID(taskInfo.RunID), tag.TaskID(taskInfo.TaskID), tag.Error(err)) - c.scope.Tagged(metrics.IsolationGroupTag(startedIsolationGroup)).IncCounter(metrics.TaskIsolationErrorPerTaskList) + c.scope.Tagged(metrics.IsolationGroupTag(startedIsolationGroup), partition.IsolationLeakCauseError).IncCounter(metrics.TaskIsolationLeakPerTaskList) return defaultTaskBufferIsolationGroup, noIsolationTimeout, nil } @@ -891,7 +892,7 @@ func (c *taskListManagerImpl) getIsolationGroupForTask(ctx context.Context, task taskIsolationDuration = totalTaskIsolationDuration - taskLatency } else { c.logger.Info("Leaking task due to taskIsolationDuration expired", tag.IsolationGroup(group), tag.IsolationDuration(taskIsolationDuration), tag.TaskLatency(taskLatency)) - c.scope.Tagged(metrics.IsolationGroupTag(group)).IncCounter(metrics.TaskIsolationExpiredPerTaskList) + c.scope.Tagged(metrics.IsolationGroupTag(startedIsolationGroup), partition.IsolationLeakCauseExpired).IncCounter(metrics.TaskIsolationLeakPerTaskList) group = defaultTaskBufferIsolationGroup } } diff --git a/service/matching/tasklist/task_list_manager_test.go b/service/matching/tasklist/task_list_manager_test.go index 462d35bf15e..faad6ca532c 100644 --- a/service/matching/tasklist/task_list_manager_test.go +++ b/service/matching/tasklist/task_list_manager_test.go @@ -221,7 +221,7 @@ func createTestTaskListManager(t *testing.T, logger log.Logger, controller *gomo func createTestTaskListManagerWithConfig(t *testing.T, logger log.Logger, controller *gomock.Controller, cfg *config.Config, timeSource clock.TimeSource) *taskListManagerImpl { tm := NewTestTaskManager(t, logger, timeSource) mockPartitioner := partition.NewMockPartitioner(controller) - mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() + mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() mockDomainCache := cache.NewMockDomainCache(controller) mockDomainCache.EXPECT().GetDomainByID(gomock.Any()).Return(cache.CreateDomainCacheEntry("domainName"), nil).AnyTimes() mockDomainCache.EXPECT().GetDomainName(gomock.Any()).Return("domainName", nil).AnyTimes() @@ -653,7 +653,7 @@ func TestGetIsolationGroupForTask(t *testing.T) { mockIsolationGroupState := isolationgroup.NewMockState(controller) mockIsolationGroupState.EXPECT().IsDrained(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() mockIsolationGroupState.EXPECT().IsDrainedByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() - mockIsolationGroupState.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, domainId string, taskListName string, available []string) (types.IsolationGroupConfiguration, error) { + mockIsolationGroupState.EXPECT().AvailableIsolationGroupsByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, domainId string, taskListName string, available []string) (types.IsolationGroupConfiguration, error) { // Report all available isolation groups as healthy isolationGroupStates := make(types.IsolationGroupConfiguration, len(available)) for _, availableGroup := range available { @@ -730,7 +730,7 @@ func TestTaskListManagerGetTaskBatch(t *testing.T) { const rangeSize = 10 controller := gomock.NewController(t) mockPartitioner := partition.NewMockPartitioner(controller) - mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() + mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() mockDomainCache := cache.NewMockDomainCache(controller) mockDomainCache.EXPECT().GetDomainByID(gomock.Any()).Return(cache.CreateDomainCacheEntry("domainName"), nil).AnyTimes() mockDomainCache.EXPECT().GetDomainName(gomock.Any()).Return("domainName", nil).AnyTimes() @@ -856,7 +856,7 @@ func TestTaskListReaderPumpAdvancesAckLevelAfterEmptyReads(t *testing.T) { controller := gomock.NewController(t) mockPartitioner := partition.NewMockPartitioner(controller) - mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() + mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() mockDomainCache := cache.NewMockDomainCache(controller) mockDomainCache.EXPECT().GetDomainByID(gomock.Any()).Return(cache.CreateDomainCacheEntry("domainName"), nil).AnyTimes() mockDomainCache.EXPECT().GetDomainName(gomock.Any()).Return("domainName", nil).AnyTimes() @@ -990,7 +990,7 @@ func TestTaskExpiryAndCompletion(t *testing.T) { t.Run("", func(t *testing.T) { controller := gomock.NewController(t) mockPartitioner := partition.NewMockPartitioner(controller) - mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() + mockPartitioner.EXPECT().GetIsolationGroupByDomainID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil).AnyTimes() mockDomainCache := cache.NewMockDomainCache(controller) mockDomainCache.EXPECT().GetDomainByID(gomock.Any()).Return(cache.CreateDomainCacheEntry("domainName"), nil).AnyTimes() mockDomainCache.EXPECT().GetDomainName(gomock.Any()).Return("domainName", nil).AnyTimes()