Consistent query pershard metric (uber#5143)

* added and update consistent query per shard metric

* testing pershard metric

* move sample logger into persistence metric client for cleaness

* fix test

* fix lint

* fix test again

* fix lint

* sample logging with workflowid tag

* added domain tag to logger

* metric completed

* addressing comments

* fix lint

* Revert "fix lint"

This reverts commit 1e96944.

* fix lint second attempt

---------

Co-authored-by: Allen Chen <allenchen2244@uber.com>

common/metrics/defs.go

@@ -2226,6 +2226,7 @@ const (
 	DirectQueryDispatchClearStickinessSuccessCount
 	DirectQueryDispatchTimeoutBeforeNonStickyCount
 	DecisionTaskQueryLatency
+	ConsistentQueryPerShard
 	ConsistentQueryTimeoutCount
 	QueryBeforeFirstDecisionCount
 	QueryBufferExceededCount
@@ -2818,6 +2819,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		DirectQueryDispatchClearStickinessSuccessCount:               {metricName: "direct_query_dispatch_clear_stickiness_success", metricType: Counter},
 		DirectQueryDispatchTimeoutBeforeNonStickyCount:               {metricName: "direct_query_dispatch_timeout_before_non_sticky", metricType: Counter},
 		DecisionTaskQueryLatency:                                     {metricName: "decision_task_query_latency", metricType: Timer},
+		ConsistentQueryPerShard:                                      {metricName: "consistent_query_per_shard", metricType: Counter},
 		ConsistentQueryTimeoutCount:                                  {metricName: "consistent_query_timeout", metricType: Counter},
 		QueryBeforeFirstDecisionCount:                                {metricName: "query_before_first_decision", metricType: Counter},
 		QueryBufferExceededCount:                                     {metricName: "query_buffer_exceeded", metricType: Counter},

service/history/historyEngine.go

@@ -27,6 +27,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/pborman/uuid"
@@ -1157,10 +1158,15 @@ func (e *historyEngineImpl) QueryWorkflow(
 ) (retResp *types.HistoryQueryWorkflowResponse, retErr error) {
 
 	scope := e.metricsClient.Scope(metrics.HistoryQueryWorkflowScope).Tagged(metrics.DomainTag(request.GetRequest().GetDomain()))
+	shardMetricScope := e.metricsClient.Scope(metrics.HistoryQueryWorkflowScope).Tagged(metrics.ShardIDTag(strconv.Itoa(e.shard.GetShardID())))
 
 	consistentQueryEnabled := e.config.EnableConsistentQuery() && e.config.EnableConsistentQueryByDomain(request.GetRequest().GetDomain())
-	if request.GetRequest().GetQueryConsistencyLevel() == types.QueryConsistencyLevelStrong && !consistentQueryEnabled {
-		return nil, workflow.ErrConsistentQueryNotEnabled
+	if request.GetRequest().GetQueryConsistencyLevel() == types.QueryConsistencyLevelStrong {
+		if !consistentQueryEnabled {
+			return nil, workflow.ErrConsistentQueryNotEnabled
+		}
+		scope.IncCounter(metrics.ConsistentQueryPerShard)
+		shardMetricScope.IncCounter(metrics.ConsistentQueryPerShard)
 	}
 
 	execution := *request.GetRequest().GetExecution()