sentry: handle "retry later" grpc stream

cl/cltypes/lightclient_test.go

@@ -1,9 +1,8 @@
 package cltypes_test
 
 import (
-	"testing"
-
 	_ "embed"
+	"testing"
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 

cmd/devnet/devnetutils/utils.go

@@ -4,15 +4,16 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
+	"math/big"
+	"os/exec"
+	"strconv"
+	"strings"
+
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon/cmd/devnet/models"
 	"github.com/ledgerwatch/erigon/cmd/rpctest/rpctest"
 	"github.com/ledgerwatch/erigon/common/hexutil"
 	"github.com/ledgerwatch/erigon/crypto"
-	"math/big"
-	"os/exec"
-	"strconv"
-	"strings"
 )
 
 // ClearDevDB cleans up the dev folder used for the operations

cmd/lightclient/lightclient/subscriber.go

@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
+	"github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/sentinel"
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -46,20 +48,35 @@ func (c *ChainTipSubscriber) StartLoop() {
 	}
 	log.Info("[LightClient Gossip] Started Gossip")
 	c.started = true
+
+Retry:
+	for {
+		if err := c.subscribeGossip(); err != nil {
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			if grpcutil.IsRetryLater(err) || grpcutil.IsEndOfStream(err) {
+				time.Sleep(3 * time.Second)
+				continue Retry
+			}
+
+			log.Warn("[Lightclient] could not read gossip :/", "reason", err)
+			time.Sleep(time.Second)
+		}
+	}
+}
+
+func (c *ChainTipSubscriber) subscribeGossip() error {
 	stream, err := c.sentinel.SubscribeGossip(c.ctx, &sentinel.EmptyMessage{})
 	if err != nil {
-		log.Warn("could not start lightclient", "reason", err)
-		return
+		return err
 	}
 	defer stream.CloseSend()
 
 	for {
 		data, err := stream.Recv()
 		if err != nil {
-			if !errors.Is(err, context.Canceled) {
-				log.Debug("[Lightclient] could not read gossip :/", "reason", err)
-			}
-			continue
+			return err
 		}
 		if err := c.handleGossipData(data); err != nil {
 			log.Warn("could not process new gossip",

cmd/release/main.go

@@ -10,8 +10,6 @@ import (
 	"sort"
 	"strings"
 	"time"
-
-	"github.com/hashicorp/go-version"
 )
 
 type Binary struct {