GOCBC-312: Add server correlation ids to ThresholdLogTracer spans

Motivation
----------
Server logs, and responses, contain correlation IDs which can be used to track
operations. The ThresholdLogTracer spans should contain these IDs so that
operations can be matched up from SDK to server.

Changes
-------
Added last_operation_id and last_local_id fields to the ThresholdLogTracer
spans. Altered n1ql and analytics stream spans to set last_operation_id.

Change-Id: I1e2f570af3565ae201b0a52abe74c3e49d4ac61e
Reviewed-on: http://review.couchbase.org/94977
Reviewed-by: Mike Goldsmith <goldsmith.mike@gmail.com>
Tested-by: Charles Dixon <chvckd@gmail.com>

cluster_analyticsquery.go

@@ -192,6 +192,7 @@ func (c *Cluster) executeAnalyticsQuery(tracectx opentracing.SpanContext, analyt
 		logDebugf("Failed to close socket (%s)", err)
 	}
 
+	strace.SetTag("couchbase.operation_id", analyticsResp.RequestId)
 	strace.Finish()
 
 	if len(analyticsResp.Errors) > 0 {
@@ -262,7 +263,7 @@ func (c *Cluster) doAnalyticsQuery(tracectx opentracing.SpanContext, q *Analytic
 // Experimental: This API is subject to change at any time.
 func (c *Cluster) ExecuteAnalyticsQuery(q *AnalyticsQuery) (AnalyticsResults, error) {
 	span := c.agentConfig.Tracer.StartSpan("ExecuteAnalyticsQuery",
-		opentracing.Tag{Key: "couchbase.service", Value: "analytics"})
+		opentracing.Tag{Key: "couchbase.service", Value: "cbas"})
 	defer span.Finish()
 
 	return c.doAnalyticsQuery(span.Context(), q)

cluster_n1qlquery.go

@@ -247,6 +247,7 @@ func (c *Cluster) executeN1qlQuery(tracectx opentracing.SpanContext, n1qlEp stri
 	//srvDuration, _ := time.ParseDuration(n1qlResp.Metrics.ExecutionTime)
 	//strace.SetTag("server_duration", srvDuration)
 
+	strace.SetTag("couchbase.operation_id", n1qlResp.RequestId)
 	strace.Finish()
 
 	if len(n1qlResp.Errors) > 0 {

thresholdlogtracer.go

@@ -83,6 +83,8 @@ type thresholdLogItem struct {
 	LastRemoteAddress      string `json:"last_remote_address,omitempty"`
 	LastLocalAddress       string `json:"last_local_address,omitempty"`
 	LastDispatchDurationUs uint64 `json:"last_dispatch_us,omitempty"`
+	LastOperationID        string `json:"last_operation_id,omitempty"`
+	LastLocalID            string `json:"last_local_id,omitempty"`
 }
 
 type thresholdLogService struct {
@@ -131,6 +133,8 @@ func (g *thresholdLogGroup) logRecordedRecords() {
 			DecodeDurationUs:       uint64(op.totalDecodeDuration / time.Microsecond),
 			LastRemoteAddress:      op.lastDispatchPeer,
 			LastDispatchDurationUs: uint64(op.lastDispatchDuration / time.Microsecond),
+			LastOperationID:        op.lastOperationID,
+			LastLocalID:            op.lastLocalID,
 		})
 	}
 
@@ -315,6 +319,8 @@ type thresholdLogSpan struct {
 	totalDecodeDuration   time.Duration
 	lastDispatchPeer      string
 	lastDispatchDuration  time.Duration
+	lastOperationID       string
+	lastLocalID           string
 }
 
 func (n *thresholdLogSpan) Context() opentracing.SpanContext {
@@ -345,6 +351,14 @@ func (n *thresholdLogSpan) SetTag(key string, value interface{}) opentracing.Spa
 		if n.peerAddress, ok = value.(string); !ok {
 			logDebugf("Failed to cast span peer.address tag")
 		}
+	case "couchbase.operation_id":
+		if n.lastOperationID, ok = value.(string); !ok {
+			logDebugf("Failed to cast span couchbase.operation_id tag")
+		}
+	case "couchbase.local_id":
+		if n.lastLocalID, ok = value.(string); !ok {
+			logDebugf("Failed to cast span couchbase.local_id tag")
+		}
 	}
 	return n
 }
@@ -382,6 +396,12 @@ func (n *thresholdLogSpan) Finish() {
 			n.parent.lastDispatchPeer = n.lastDispatchPeer
 			n.parent.lastDispatchDuration = n.lastDispatchDuration
 		}
+		if n.lastOperationID != "" {
+			n.parent.lastOperationID = n.lastOperationID
+		}
+		if n.lastLocalID != "" {
+			n.parent.lastLocalID = n.lastLocalID
+		}
 	}
 
 	if n.serviceName != "" {