etcd-tester: add retry logic on retriving lease info

getting lease and keys info through raw rpcs rarely experience error such as EOF. This is considered as a failure and causes tester to clean up.
however, they are just transient problem wih temporary connection issue which should not considered as a testing failure. so we add retry logic in case of transient failure.

FIX #6754

tools/functional-tester/etcd-tester/lease_stresser.go

@@ -258,28 +258,35 @@ func (ls *leaseStresser) getLeaseByID(ctx context.Context, leaseID int64) (*pb.L
 }
 
 func (ls *leaseStresser) hasLeaseExpired(ctx context.Context, leaseID int64) (bool, error) {
-	resp, err := ls.getLeaseByID(ctx, leaseID)
-	plog.Debugf("hasLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
-	if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound {
-		return true, nil
+	for ctx.Err() == nil {
+		resp, err := ls.getLeaseByID(ctx, leaseID)
+		plog.Debugf("hasLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
+		if err == nil {
+			return false, nil
+		}
+		if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound {
+			return true, nil
+		}
 	}
-	return false, err
+	return false, ctx.Err()
 }
 
 // The keys attached to the lease has the format of "<leaseID>_<idx>" where idx is the ordering key creation
 // Since the format of keys contains about leaseID, finding keys base on "<leaseID>" prefix
 // determines whether the attached keys for a given leaseID has been deleted or not
 func (ls *leaseStresser) hasKeysAttachedToLeaseExpired(ctx context.Context, leaseID int64) (bool, error) {
-	resp, err := ls.kvc.Range(ctx, &pb.RangeRequest{
-		Key:      []byte(fmt.Sprintf("%d", leaseID)),
-		RangeEnd: []byte(clientv3.GetPrefixRangeEnd(fmt.Sprintf("%d", leaseID))),
-	}, grpc.FailFast(false))
-	plog.Debugf("hasKeysAttachedToLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
-	if err != nil {
-		plog.Errorf("retriving keys attached to lease %v error: (%v)", leaseID, err)
-		return false, err
+	for ctx.Err() == nil {
+		resp, err := ls.kvc.Range(ctx, &pb.RangeRequest{
+			Key:      []byte(fmt.Sprintf("%d", leaseID)),
+			RangeEnd: []byte(clientv3.GetPrefixRangeEnd(fmt.Sprintf("%d", leaseID))),
+		}, grpc.FailFast(false))
+		plog.Debugf("hasKeysAttachedToLeaseExpired %v resp %v error (%v)", leaseID, resp, err)
+		if err == nil {
+			return len(resp.Kvs) == 0, nil
+		}
+		plog.Warningf("retriving keys attached to lease %v error: (%v)", leaseID, err)
 	}
-	return len(resp.Kvs) == 0, nil
+	return false, ctx.Err()
 }
 
 func (ls *leaseStresser) createLease(ttl int64) (int64, error) {