profiler: collect peer latency and tunnel types

Author: zllovesuki

gateway/gateway.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/zllovesuki/t/multiplexer"
 	"github.com/zllovesuki/t/multiplexer/alpn"
+	"github.com/zllovesuki/t/profiler"
 	"github.com/zllovesuki/t/server"
 	"github.com/zllovesuki/t/shared"
 
@@ -93,19 +94,27 @@ func (g *Gateway) handleConnection(ctx context.Context, conn *tls.Conn) {
 		switch cs.NegotiatedProtocol {
 		case alpn.Unknown.String(), alpn.HTTP.String():
 			g.Logger.Debug("forward http connection")
+			profiler.GatewayReqsType.WithLabelValues("http").Inc()
+
 			g.httpTunnelAcceptor.ch <- conn
 		case alpn.Raw.String():
 			g.Logger.Debug("forward raw connection")
+			profiler.GatewayReqsType.WithLabelValues("raw").Inc()
+
 			_, err := g.Multiplexer.Forward(ctx, conn, g.link(cs.ServerName, cs.NegotiatedProtocol))
 			if err != nil {
 				g.Logger.Error("establish raw link error", zap.Error(err))
 				conn.Close()
 			}
 		case alpn.Multiplexer.String():
 			g.Logger.Error("received alpn proposal for multiplexer on gateway")
+			profiler.GatewayReqsType.WithLabelValues("multiplexer").Inc()
+
 			conn.Close()
 		default:
 			g.Logger.Error("unknown alpn proposal", zap.String("proposal", cs.NegotiatedProtocol))
+			profiler.GatewayReqsType.WithLabelValues("error").Inc()
+
 			conn.Close()
 		}
 	}

gateway/tunnel_http.go

@@ -25,7 +25,8 @@ func (g *Gateway) httpHandler() http.Handler {
 		},
 		Transport: &http.Transport{
 			DialContext: func(c context.Context, network, addr string) (net.Conn, error) {
-				return g.Multiplexer.Direct(c, g.link(addr, alpn.HTTP.String()))
+				conn, err := g.Multiplexer.Direct(c, g.link(addr, alpn.HTTP.String()))
+				return conn, err
 			},
 			// TODO(zllovesuki): Make MaxConnsPerHost configurable
 			MaxConnsPerHost:       15,

profiler/metrics.go

@@ -10,10 +10,24 @@ var (
 		Name:      "gateway_requests_total",
 	}, []string{"status", "type"})
 
+	GatewayReqsType = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "t",
+		Subsystem: "server",
+		Help:      "Count of all forwarded tunnel types",
+		Name:      "gateway_tunnel_type",
+	}, []string{"type"})
+
 	ConnectionStats = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "t",
 		Subsystem: "server",
 		Help:      "Stats on multiplexer connections",
 		Name:      "multiplexer_stats",
 	}, []string{"desc", "type"})
+
+	PeerLatencyHist = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "t",
+		Subsystem: "server",
+		Help:      "Histogram of peers latency via gossip",
+		Name:      "peer_rtt_ms",
+	})
 )

profiler/profiler.go

@@ -12,7 +12,9 @@ func StartProfiler(addr string) {
 
 	r := prometheus.NewRegistry()
 	r.MustRegister(GatewayRequests)
+	r.MustRegister(GatewayReqsType)
 	r.MustRegister(ConnectionStats)
+	r.MustRegister(PeerLatencyHist)
 
 	m.Handle("/", debug())
 	m.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))

server/gossip.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zllovesuki/t/acme"
 	"github.com/zllovesuki/t/multiplexer"
 	"github.com/zllovesuki/t/multiplexer/alpn"
+	"github.com/zllovesuki/t/profiler"
 	"github.com/zllovesuki/t/state"
 
 	"github.com/hashicorp/memberlist"
@@ -218,8 +219,10 @@ func (s *Server) NotifyPingComplete(other *memberlist.Node, rtt time.Duration, p
 		s.logger.Error("unmarshal node meta from ping", zap.Error(err))
 		return
 	}
+
 	n := binary.BigEndian.Uint64(payload)
-	s.logger.Info("ping", zap.Uint64("Peer", m.PeerID), zap.Duration("rtt", rtt), zap.Uint64("numPeers", n))
+	s.logger.Debug("ping", zap.Uint64("Peer", m.PeerID), zap.Duration("rtt", rtt), zap.Uint64("numPeers", n))
+	profiler.PeerLatencyHist.Observe(float64(rtt / time.Millisecond))
 }
 
 // ======== Gossip Helpers ========