RpcDaemon doesn't see recently retired blocks

erigon-lib/kv/remotedbserver/remotedbserver.go

@@ -458,17 +458,27 @@ func (s *KvServer) SendStateChanges(_ context.Context, sc *remote.StateChangeBat
 	s.stateChangeStreams.Pub(sc)
 }
 
-func (s *KvServer) Snapshots(_ context.Context, _ *remote.SnapshotsRequest) (*remote.SnapshotsReply, error) {
+func (s *KvServer) Snapshots(_ context.Context, _ *remote.SnapshotsRequest) (reply *remote.SnapshotsReply, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = fmt.Errorf("%v, %s", rec, dbg.Stack())
+		}
+	}()
 	if s.blockSnapshots == nil || reflect.ValueOf(s.blockSnapshots).IsNil() { // nolint
 		return &remote.SnapshotsReply{BlocksFiles: []string{}, HistoryFiles: []string{}}, nil
 	}
 
 	blockFiles := s.blockSnapshots.Files()
-	if s.borSnapshots != nil {
+	if s.borSnapshots != nil && !reflect.ValueOf(s.borSnapshots).IsNil() { // nolint
 		blockFiles = append(blockFiles, s.borSnapshots.Files()...)
 	}
 
-	return &remote.SnapshotsReply{BlocksFiles: blockFiles, HistoryFiles: s.historySnapshots.Files()}, nil
+	reply = &remote.SnapshotsReply{BlocksFiles: blockFiles}
+	if s.historySnapshots != nil && !reflect.ValueOf(s.historySnapshots).IsNil() { // nolint
+		reply.HistoryFiles = s.historySnapshots.Files()
+	}
+
+	return reply, nil
 }
 
 type StateChangePubSub struct {

eth/integrity/snap_blocks_read.go

@@ -0,0 +1,46 @@
+package integrity
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/turbo/services"
+	"github.com/ledgerwatch/log/v3"
+)
+
+func SnapBlocksRead(db kv.RoDB, blockReader services.FullBlockReader, ctx context.Context, failFast bool) error {
+	defer log.Info("[integrity] SnapBlocksRead: done")
+	logEvery := time.NewTicker(10 * time.Second)
+	defer logEvery.Stop()
+
+	maxBlockNum := blockReader.Snapshots().SegmentsMax()
+	for i := uint64(0); i < maxBlockNum; i += 10_000 {
+		if err := db.View(ctx, func(tx kv.Tx) error {
+			b, err := blockReader.BlockByNumber(ctx, tx, i)
+			if err != nil {
+				return err
+			}
+			if b == nil {
+				err := fmt.Errorf("block not found in snapshots: %d\n", i)
+				if failFast {
+					return err
+				}
+				log.Error("[integrity] SnapBlocksRead", "err", err)
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-logEvery.C:
+			log.Info("[integrity] SnapBlocksRead", "blockNum", fmt.Sprintf("%dK/%dK", i/1000, maxBlockNum/1000))
+		default:
+		}
+	}
+	return nil
+}

turbo/app/snapshots_cmd.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/erigon-lib/common/dir"
 	"github.com/ledgerwatch/erigon-lib/metrics"
+	"github.com/ledgerwatch/erigon/eth/integrity"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sync/semaphore"
@@ -223,10 +224,14 @@ func doIntegrity(cliCtx *cli.Context) error {
 	defer agg.Close()
 
 	blockReader, _ := blockRetire.IO()
-	if err := blockReader.(*freezeblocks.BlockReader).IntegrityTxnID(false); err != nil {
+	if err := integrity.SnapBlocksRead(chainDB, blockReader, ctx, false); err != nil {
 		return err
 	}
 
+	//if err := blockReader.(*freezeblocks.BlockReader).IntegrityTxnID(false); err != nil {
+	//	return err
+	//}
+
 	//if err := integrity.E3HistoryNoSystemTxs(ctx, chainDB, agg); err != nil {
 	//	return err
 	//}