improve client RPC metrics consistency (#19721)

The client.rpc metric now excludes internal retries for consistency
with client.rpc.exceeded and client.rpc.failed. All of these metrics
now increment at most once per RPC method call, allowing for
accurate calculation of failure / rate limit application occurrence.

Additionally, if an RPC fails because no servers are present,
client.rpc.failed is now incremented.

.changelog/19721.txt

@@ -0,0 +1,6 @@
+```release-note:improvement
+metrics: modify consul.client.rpc metric to exclude internal retries for consistency with consul.client.rpc.exceeded and consul.client.rpc.failed
+```
+```release-note:improvement
+metrics: increment consul.client.rpc.failed if RPC fails because no servers are accessible
+```

agent/consul/client.go

@@ -288,15 +288,17 @@ func (c *Client) RPC(ctx context.Context, method string, args interface{}, reply
 	firstCheck := time.Now()
 	retryCount := 0
 	previousJitter := time.Duration(0)
+
+	metrics.IncrCounter([]string{"client", "rpc"}, 1)
 TRY:
 	retryCount++
 	manager, server := c.router.FindLANRoute()
 	if server == nil {
+		metrics.IncrCounter([]string{"client", "rpc", "failed"}, 1)
 		return structs.ErrNoServers
 	}
 
 	// Enforce the RPC limit.
-	metrics.IncrCounter([]string{"client", "rpc"}, 1)
 	if !c.rpcLimiter.Load().(*rate.Limiter).Allow() {
 		metrics.IncrCounter([]string{"client", "rpc", "exceeded"}, 1)
 		return structs.ErrRPCRateExceeded