sentry: handle "retry later" grpc stream (#6852)

Author: AskAlexSharov

cl/cltypes/lightclient_test.go

@@ -1,9 +1,8 @@
 package cltypes_test
 
 import (
-	"testing"
-
 	_ "embed"
+	"testing"
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 

cmd/lightclient/lightclient/subscriber.go

@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
+	"github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/sentinel"
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -46,20 +48,35 @@ func (c *ChainTipSubscriber) StartLoop() {
 	}
 	log.Info("[LightClient Gossip] Started Gossip")
 	c.started = true
+
+Retry:
+	for {
+		if err := c.subscribeGossip(); err != nil {
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			if grpcutil.IsRetryLater(err) || grpcutil.IsEndOfStream(err) {
+				time.Sleep(3 * time.Second)
+				continue Retry
+			}
+
+			log.Warn("[Lightclient] could not read gossip :/", "reason", err)
+			time.Sleep(time.Second)
+		}
+	}
+}
+
+func (c *ChainTipSubscriber) subscribeGossip() error {
 	stream, err := c.sentinel.SubscribeGossip(c.ctx, &sentinel.EmptyMessage{})
 	if err != nil {
-		log.Warn("could not start lightclient", "reason", err)
-		return
+		return err
 	}
 	defer stream.CloseSend()
 
 	for {
 		data, err := stream.Recv()
 		if err != nil {
-			if !errors.Is(err, context.Canceled) {
-				log.Debug("[Lightclient] could not read gossip :/", "reason", err)
-			}
-			continue
+			return err
 		}
 		if err := c.handleGossipData(data); err != nil {
 			log.Warn("could not process new gossip",