fix unexpected score in look aside balancer

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

internal/proxy/look_aside_balancer.go

@@ -159,8 +159,20 @@ func (b *LookAsideBalancer) calculateScore(node int64, cost *internalpb.CostAggr
 	}
 
 	executeSpeed := float64(cost.ResponseTime) - float64(cost.ServiceTime)
-	workload := math.Pow(float64(1+cost.TotalNQ+executingNQ), 3.0) * float64(cost.ServiceTime)
-	if workload < 0.0 {
+	if executingNQ < 0 {
+		log.Warn("unexpected executing nq value",
+			zap.Int64("executingNQ", executingNQ))
+		return executeSpeed
+	}
+
+	if cost.GetTotalNQ() < 0 {
+		log.Warn("unexpected total nq value",
+			zap.Int64("totalNq", cost.GetTotalNQ()))
+		return executeSpeed
+	}
+
+	workload := math.Pow(float64(1+cost.GetTotalNQ()+executingNQ), 3.0) * float64(cost.ServiceTime)
+	if workload < 0 {
 		return math.MaxFloat64
 	}
 

internal/proxy/look_aside_balancer_test.go

@@ -128,6 +128,22 @@ func (suite *LookAsideBalancerSuite) TestCalculateScore() {
 	suite.balancer.metricsUpdateTs.Insert(1, time.Now().UnixMilli()-5000)
 	score11 := suite.balancer.calculateScore(1, costMetrics4, 0)
 	suite.Equal(float64(0), score11)
+
+	// test unexpected negative nq value
+	costMetrics6 := &internalpb.CostAggregation{
+		ResponseTime: 5,
+		ServiceTime:  1,
+		TotalNQ:      -1,
+	}
+	score12 := suite.balancer.calculateScore(-1, costMetrics6, math.MaxInt64)
+	suite.Equal(float64(4), score12)
+	costMetrics7 := &internalpb.CostAggregation{
+		ResponseTime: 5,
+		ServiceTime:  1,
+		TotalNQ:      1,
+	}
+	score13 := suite.balancer.calculateScore(-1, costMetrics7, -1)
+	suite.Equal(float64(4), score13)
 }
 
 func (suite *LookAsideBalancerSuite) TestSelectNode() {

pkg/metrics/proxy_metrics.go

@@ -330,6 +330,7 @@ func RegisterProxy(registry *prometheus.Registry) {
 	registry.MustRegister(UserRPCCounter)
 
 	registry.MustRegister(ProxyWorkLoadScore)
+	registry.MustRegister(ProxyExecutingTotalNq)
 }
 
 func CleanupCollectionMetrics(nodeID int64, collection string) {