From 6904e4c3c48b9dcab6df35b73b98e57eeb1d3cd8 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Fri, 31 Dec 2021 20:42:56 +0700 Subject: [PATCH] Snapshot: describe idx schema (#3191) --- cmd/downloader/downloader/downloader.go | 19 +-- cmd/downloader/downloader/server.go | 6 +- cmd/downloader/downloader/util.go | 1 + cmd/downloader/main.go | 42 ++++--- cmd/downloader/readme.md | 1 - eth/stagedsync/stage_headers.go | 2 +- go.mod | 2 +- go.sum | 4 +- turbo/snapshotsync/block_reader.go | 32 +++--- turbo/snapshotsync/block_snapshots.go | 146 +++++++++--------------- 10 files changed, 108 insertions(+), 147 deletions(-) diff --git a/cmd/downloader/downloader/downloader.go b/cmd/downloader/downloader/downloader.go index 42c0ebb3e08..02d13e1c94e 100644 --- a/cmd/downloader/downloader/downloader.go +++ b/cmd/downloader/downloader/downloader.go @@ -3,7 +3,6 @@ package downloader import ( "context" "fmt" - "path/filepath" "runtime" "sync" "time" @@ -17,11 +16,12 @@ import ( "github.com/dustin/go-humanize" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/common" - "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes" "github.com/ledgerwatch/log/v3" "golang.org/x/time/rate" ) +const ASSERT = false + type Client struct { Cli *torrent.Client pieceCompletionStore storage.PieceCompletion @@ -231,8 +231,7 @@ func calcStats(prevStats aggStats, interval time.Duration, client *torrent.Clien // added first time - pieces verification process will start (disk IO heavy) - progress // kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. // Don't need call torrent.VerifyData manually -func AddTorrentFiles(ctx context.Context, snapshotsDir string, torrentClient *torrent.Client, preverifiedHashes snapshothashes.Preverified) error { - +func AddTorrentFiles(ctx context.Context, snapshotsDir string, torrentClient *torrent.Client) error { if err := ForEachTorrentFile(snapshotsDir, func(torrentFilePath string) error { mi, err := metainfo.LoadFromFile(torrentFilePath) if err != nil { @@ -240,18 +239,6 @@ func AddTorrentFiles(ctx context.Context, snapshotsDir string, torrentClient *to } mi.AnnounceList = Trackers - // skip non-preverified files - _, torrentFileName := filepath.Split(torrentFilePath) - segmentFileName := segmentFileNameFromTorrentFileName(torrentFileName) - hashString, ok := preverifiedHashes[segmentFileName] - if !ok { - return nil - } - expect := metainfo.NewHashFromHex(hashString) - if mi.HashInfoBytes() != expect { - return fmt.Errorf("file %s has unexpected hash %x, expected %x. May help: git submodule update --init --recursive --force", torrentFileName, mi.HashInfoBytes(), expect) - } - if _, err = torrentClient.AddTorrent(mi); err != nil { return err } diff --git a/cmd/downloader/downloader/server.go b/cmd/downloader/downloader/server.go index 44cd15c75eb..d82c5fda7cc 100644 --- a/cmd/downloader/downloader/server.go +++ b/cmd/downloader/downloader/server.go @@ -11,7 +11,6 @@ import ( proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes" "google.golang.org/protobuf/types/known/emptypb" ) @@ -32,11 +31,11 @@ func NewServer(db kv.RwDB, client *Client, snapshotDir string) (*SNDownloaderSer return sn, nil } -func CreateTorrentFilesAndAdd(ctx context.Context, snapshotDir string, torrentClient *torrent.Client, config *snapshothashes.Config) error { +func CreateTorrentFilesAndAdd(ctx context.Context, snapshotDir string, torrentClient *torrent.Client) error { if err := BuildTorrentFilesIfNeed(ctx, snapshotDir); err != nil { return err } - if err := AddTorrentFiles(ctx, snapshotDir, torrentClient, config.Preverified); err != nil { + if err := AddTorrentFiles(ctx, snapshotDir, torrentClient); err != nil { return err } for _, t := range torrentClient.Torrents() { @@ -44,7 +43,6 @@ func CreateTorrentFilesAndAdd(ctx context.Context, snapshotDir string, torrentCl t.AllowDataUpload() t.DownloadAll() } - return nil } diff --git a/cmd/downloader/downloader/util.go b/cmd/downloader/downloader/util.go index cc4bf3c9c63..e11e4371e53 100644 --- a/cmd/downloader/downloader/util.go +++ b/cmd/downloader/downloader/util.go @@ -173,6 +173,7 @@ func CreateTorrentFile(root string, info *metainfo.Info, mi *metainfo.MetaInfo) return nil } +// nolint func segmentFileNameFromTorrentFileName(in string) string { ext := filepath.Ext(in) return in[0 : len(in)-len(ext)] diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 459ef40dfb4..45789e9758d 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -153,26 +153,36 @@ func Downloader(ctx context.Context, cmd *cobra.Command) error { return fmt.Errorf("new server: %w", err) } - var cc *params.ChainConfig - { - chaindataDir := path.Join(datadir, "chaindata") - if err := os.MkdirAll(chaindataDir, 0755); err != nil { - return err - } - chaindata, err := mdbx.Open(chaindataDir, log.New(), true) - if err != nil { - return fmt.Errorf("%w, path: %s", err, chaindataDir) - } - cc = tool.ChainConfigFromDB(chaindata) - chaindata.Close() - } - - snapshotsCfg := snapshothashes.KnownConfig(cc.ChainName) - err = downloader.CreateTorrentFilesAndAdd(ctx, snapshotsDir, t.Cli, snapshotsCfg) + err = downloader.CreateTorrentFilesAndAdd(ctx, snapshotsDir, t.Cli) if err != nil { return fmt.Errorf("start: %w", err) } + if downloader.ASSERT { + var cc *params.ChainConfig + { + chaindataDir := path.Join(datadir, "chaindata") + if err := os.MkdirAll(chaindataDir, 0755); err != nil { + return err + } + chaindata, err := mdbx.Open(chaindataDir, log.New(), true) + if err != nil { + return fmt.Errorf("%w, path: %s", err, chaindataDir) + } + cc = tool.ChainConfigFromDB(chaindata) + chaindata.Close() + } + + snapshotsCfg := snapshothashes.KnownConfig(cc.ChainName) + for _, t := range t.Cli.Torrents() { + expectHashStr := snapshotsCfg.Preverified[t.Info().Name] + expectHash := metainfo.NewHashFromHex(expectHashStr) + if t.InfoHash() != expectHash { + return fmt.Errorf("file %s has unexpected hash %x, expected %x. May help: git submodule update --init --recursive --force", t.Info().Name, t.InfoHash(), expectHash) + } + } + } + go downloader.MainLoop(ctx, t.Cli) grpcServer, err := StartGrpc(bittorrentServer, downloaderApiAddr, nil) diff --git a/cmd/downloader/readme.md b/cmd/downloader/readme.md index dc1f282ae82..91cc988b602 100644 --- a/cmd/downloader/readme.md +++ b/cmd/downloader/readme.md @@ -61,4 +61,3 @@ rsync server1:/snapshots/*.torrent server2:/snapshot // re-start downloader ``` - diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go index 55b165a49f4..f97f75afd85 100644 --- a/eth/stagedsync/stage_headers.go +++ b/eth/stagedsync/stage_headers.go @@ -847,7 +847,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R } // ResetSequence - allow set arbitrary value to sequence (for example to decrement it to exact value) - lastTxnID := sn.Transactions.Idx.BaseDataID() + uint64(sn.Transactions.Segment.Count()) + lastTxnID := sn.TxnHashIdx.BaseDataID() + uint64(sn.Transactions.Count()) if err := rawdb.ResetSequence(tx, kv.EthTx, lastTxnID+1); err != nil { return err } diff --git a/go.mod b/go.mod index dc56439f70c..01017dcc409 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/kevinburke/go-bindata v3.21.0+incompatible - github.com/ledgerwatch/erigon-lib v0.0.0-20211228005012-bd5e7706f075 + github.com/ledgerwatch/erigon-lib v0.0.0-20211231112434-5f40a789859a github.com/ledgerwatch/log/v3 v3.4.0 github.com/ledgerwatch/secp256k1 v1.0.0 github.com/logrusorgru/aurora/v3 v3.0.0 diff --git a/go.sum b/go.sum index 6df5a17df62..63942c71801 100644 --- a/go.sum +++ b/go.sum @@ -619,8 +619,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20211228005012-bd5e7706f075 h1:YlrZJZxE5AaczZ8r5ECgSXvI0tuGJEDPCxCyKCs26jQ= -github.com/ledgerwatch/erigon-lib v0.0.0-20211228005012-bd5e7706f075/go.mod h1:lyGP3i0x4CeabdKZ4beycD5xZfHWZwJsAX+70OfGj4Y= +github.com/ledgerwatch/erigon-lib v0.0.0-20211231112434-5f40a789859a h1:LWNNoeLpGSrnR8N7McTKjPHgfzgt6YToyBfr8fy1Z1w= +github.com/ledgerwatch/erigon-lib v0.0.0-20211231112434-5f40a789859a/go.mod h1:lyGP3i0x4CeabdKZ4beycD5xZfHWZwJsAX+70OfGj4Y= github.com/ledgerwatch/log/v3 v3.4.0 h1:SEIOcv5a2zkG3PmoT5jeTU9m/0nEUv0BJS5bzsjwKCI= github.com/ledgerwatch/log/v3 v3.4.0/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/turbo/snapshotsync/block_reader.go b/turbo/snapshotsync/block_reader.go index 5ad1676eb89..134b2ae28f9 100644 --- a/turbo/snapshotsync/block_reader.go +++ b/turbo/snapshotsync/block_reader.go @@ -232,10 +232,10 @@ func (back *BlockReaderWithSnapshots) BlockWithSenders(ctx context.Context, tx k back.lock.Lock() defer back.lock.Unlock() - headerOffset := sn.Headers.Idx.Lookup2(blockHeight - sn.Headers.Idx.BaseDataID()) - bodyOffset := sn.Bodies.Idx.Lookup2(blockHeight - sn.Bodies.Idx.BaseDataID()) + headerOffset := sn.HeaderHashIdx.Lookup2(blockHeight - sn.HeaderHashIdx.BaseDataID()) + bodyOffset := sn.BodyNumberIdx.Lookup2(blockHeight - sn.BodyNumberIdx.BaseDataID()) - gg := sn.Headers.Segment.MakeGetter() + gg := sn.Headers.MakeGetter() gg.Reset(headerOffset) buf, _ = gg.Next(buf[:0]) h := &types.Header{} @@ -243,7 +243,7 @@ func (back *BlockReaderWithSnapshots) BlockWithSenders(ctx context.Context, tx k return nil, nil, err } - gg = sn.Bodies.Segment.MakeGetter() + gg = sn.Bodies.MakeGetter() gg.Reset(bodyOffset) buf, _ = gg.Next(buf[:0]) b := &types.BodyForStorage{} @@ -252,15 +252,15 @@ func (back *BlockReaderWithSnapshots) BlockWithSenders(ctx context.Context, tx k return nil, nil, err } - if b.BaseTxId < sn.Transactions.Idx.BaseDataID() { - return nil, nil, fmt.Errorf(".idx file has wrong baseDataID? %d<%d, %s", b.BaseTxId, sn.Transactions.Idx.BaseDataID(), sn.Transactions.File) + if b.BaseTxId < sn.TxnHashIdx.BaseDataID() { + return nil, nil, fmt.Errorf(".idx file has wrong baseDataID? %d<%d, %s", b.BaseTxId, sn.TxnHashIdx.BaseDataID(), sn.Transactions.FilePath()) } txs := make([]types.Transaction, b.TxAmount) senders = make([]common.Address, b.TxAmount) if b.TxAmount > 0 { - txnOffset := sn.Transactions.Idx.Lookup2(b.BaseTxId - sn.Transactions.Idx.BaseDataID()) // need subtract baseID of indexFile - gg = sn.Transactions.Segment.MakeGetter() + txnOffset := sn.TxnHashIdx.Lookup2(b.BaseTxId - sn.TxnHashIdx.BaseDataID()) // need subtract baseID of indexFile + gg = sn.Transactions.MakeGetter() gg.Reset(txnOffset) stream := rlp.NewStream(reader, 0) for i := uint32(0); i < b.TxAmount; i++ { @@ -287,8 +287,8 @@ func (back *BlockReaderWithSnapshots) BlockWithSenders(ctx context.Context, tx k func (back *BlockReaderWithSnapshots) headerFromSnapshot(blockHeight uint64, sn *BlocksSnapshot) (*types.Header, error) { buf := make([]byte, 16) - headerOffset := sn.Headers.Idx.Lookup2(blockHeight - sn.Headers.Idx.BaseDataID()) - gg := sn.Headers.Segment.MakeGetter() + headerOffset := sn.HeaderHashIdx.Lookup2(blockHeight - sn.HeaderHashIdx.BaseDataID()) + gg := sn.Headers.MakeGetter() gg.Reset(headerOffset) buf, _ = gg.Next(buf[:0]) h := &types.Header{} @@ -301,9 +301,9 @@ func (back *BlockReaderWithSnapshots) headerFromSnapshot(blockHeight uint64, sn func (back *BlockReaderWithSnapshots) bodyFromSnapshot(blockHeight uint64, sn *BlocksSnapshot) (*types.Body, []common.Address, uint64, uint32, error) { buf := make([]byte, 16) - bodyOffset := sn.Bodies.Idx.Lookup2(blockHeight - sn.Bodies.Idx.BaseDataID()) + bodyOffset := sn.BodyNumberIdx.Lookup2(blockHeight - sn.BodyNumberIdx.BaseDataID()) - gg := sn.Bodies.Segment.MakeGetter() + gg := sn.Bodies.MakeGetter() gg.Reset(bodyOffset) buf, _ = gg.Next(buf[:0]) b := &types.BodyForStorage{} @@ -312,15 +312,15 @@ func (back *BlockReaderWithSnapshots) bodyFromSnapshot(blockHeight uint64, sn *B return nil, nil, 0, 0, err } - if b.BaseTxId < sn.Transactions.Idx.BaseDataID() { - return nil, nil, 0, 0, fmt.Errorf(".idx file has wrong baseDataID? %d<%d, %s", b.BaseTxId, sn.Transactions.Idx.BaseDataID(), sn.Transactions.File) + if b.BaseTxId < sn.TxnHashIdx.BaseDataID() { + return nil, nil, 0, 0, fmt.Errorf(".idx file has wrong baseDataID? %d<%d, %s", b.BaseTxId, sn.TxnHashIdx.BaseDataID(), sn.Transactions.FilePath()) } txs := make([]types.Transaction, b.TxAmount) senders := make([]common.Address, b.TxAmount) if b.TxAmount > 0 { - txnOffset := sn.Transactions.Idx.Lookup2(b.BaseTxId - sn.Transactions.Idx.BaseDataID()) // need subtract baseID of indexFile - gg = sn.Transactions.Segment.MakeGetter() + txnOffset := sn.TxnHashIdx.Lookup2(b.BaseTxId - sn.TxnHashIdx.BaseDataID()) // need subtract baseID of indexFile + gg = sn.Transactions.MakeGetter() gg.Reset(txnOffset) stream := rlp.NewStream(reader, 0) for i := uint32(0); i < b.TxAmount; i++ { diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/block_snapshots.go index 166eed80437..90cd66879ad 100644 --- a/turbo/snapshotsync/block_snapshots.go +++ b/turbo/snapshotsync/block_snapshots.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "crypto/sha256" "encoding/binary" "errors" "fmt" @@ -35,6 +34,18 @@ import ( "github.com/ledgerwatch/log/v3" ) +type BlocksSnapshot struct { + Bodies *compress.Decompressor // value: rlp(types.BodyForStorage) + Headers *compress.Decompressor // value: header_rlp + Transactions *compress.Decompressor // value: first_byte_of_transaction_hash + transaction_rlp + BodyNumberIdx *recsplit.Index // block_num_u64 -> bodies_segment_offset + HeaderHashIdx *recsplit.Index // header_hash -> headers_segment_offset + TxnHashIdx *recsplit.Index // transaction_hash -> transactions_segment_offset + + From uint64 // included + To uint64 // excluded +} + type SnapshotType string const ( @@ -65,24 +76,6 @@ func IdxFileName(from, to uint64, name SnapshotType) string { return FileName(from, to, name) + ".idx" } -type Snapshot struct { - File string - Idx *recsplit.Index - Segment *compress.Decompressor - From uint64 // included - To uint64 // excluded -} - -func (s Snapshot) Has(block uint64) bool { return block >= s.From && block < s.To } - -type BlocksSnapshot struct { - Bodies *Snapshot - Headers *Snapshot - Transactions *Snapshot - From uint64 // included - To uint64 // excluded -} - func (s BlocksSnapshot) Has(block uint64) bool { return block >= s.From && block < s.To } type AllSnapshots struct { @@ -149,43 +142,43 @@ func (s *AllSnapshots) ReopenSomeIndices(types ...SnapshotType) error { for _, snapshotType := range types { switch snapshotType { case Headers: - if bs.Headers.Idx != nil { - bs.Headers.Idx.Close() - bs.Headers.Idx = nil + if bs.HeaderHashIdx != nil { + bs.HeaderHashIdx.Close() + bs.HeaderHashIdx = nil } - idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Headers.From, bs.Headers.To, Headers))) + idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.From, bs.To, Headers))) if err != nil { return err } - bs.Headers.Idx = idx + bs.HeaderHashIdx = idx case Bodies: - if bs.Bodies.Idx != nil { - bs.Bodies.Idx.Close() - bs.Bodies.Idx = nil + if bs.BodyNumberIdx != nil { + bs.BodyNumberIdx.Close() + bs.BodyNumberIdx = nil } - idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Bodies.From, bs.Bodies.To, Bodies))) + idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.From, bs.To, Bodies))) if err != nil { return err } - bs.Bodies.Idx = idx + bs.BodyNumberIdx = idx case Transactions: - if bs.Transactions.Idx != nil { - bs.Transactions.Idx.Close() - bs.Transactions.Idx = nil + if bs.TxnHashIdx != nil { + bs.TxnHashIdx.Close() + bs.TxnHashIdx = nil } - idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.Transactions.From, bs.Transactions.To, Transactions))) + idx, err := recsplit.OpenIndex(path.Join(s.dir, IdxFileName(bs.From, bs.To, Transactions))) if err != nil { return err } - bs.Transactions.Idx = idx + bs.TxnHashIdx = idx default: panic(fmt.Sprintf("unknown snapshot type: %s", snapshotType)) } } - if bs.Transactions.To > 0 { - s.idxAvailable = bs.Transactions.To - 1 + if bs.To > 0 { + s.idxAvailable = bs.To - 1 } else { s.idxAvailable = 0 } @@ -225,36 +218,33 @@ func (s *AllSnapshots) ReopenSegments() error { blocksSnapshot := &BlocksSnapshot{From: from, To: to} { fileName := SegmentFileName(from, to, Bodies) - d, err := compress.NewDecompressor(path.Join(dir, fileName)) + blocksSnapshot.Bodies, err = compress.NewDecompressor(path.Join(dir, fileName)) if err != nil { if errors.Is(err, os.ErrNotExist) { break } return err } - blocksSnapshot.Bodies = &Snapshot{From: from, To: to, File: path.Join(dir, fileName), Segment: d} } { fileName := SegmentFileName(from, to, Headers) - d, err := compress.NewDecompressor(path.Join(dir, fileName)) + blocksSnapshot.Headers, err = compress.NewDecompressor(path.Join(dir, fileName)) if err != nil { if errors.Is(err, os.ErrNotExist) { break } return err } - blocksSnapshot.Headers = &Snapshot{From: from, To: to, File: path.Join(dir, fileName), Segment: d} } { fileName := SegmentFileName(from, to, Transactions) - d, err := compress.NewDecompressor(path.Join(dir, fileName)) + blocksSnapshot.Transactions, err = compress.NewDecompressor(path.Join(dir, fileName)) if err != nil { if errors.Is(err, os.ErrNotExist) { break } return err } - blocksSnapshot.Transactions = &Snapshot{From: from, To: to, File: path.Join(dir, fileName), Segment: d} } s.blocks = append(s.blocks, blocksSnapshot) @@ -269,23 +259,23 @@ func (s *AllSnapshots) ReopenSegments() error { func (s *AllSnapshots) Close() { for _, s := range s.blocks { - if s.Headers.Idx != nil { - s.Headers.Idx.Close() + if s.HeaderHashIdx != nil { + s.HeaderHashIdx.Close() } - if s.Headers.Segment != nil { - s.Headers.Segment.Close() + if s.Headers != nil { + s.Headers.Close() } - if s.Bodies.Idx != nil { - s.Bodies.Idx.Close() + if s.BodyNumberIdx != nil { + s.BodyNumberIdx.Close() } - if s.Bodies.Segment != nil { - s.Bodies.Segment.Close() + if s.Bodies != nil { + s.Bodies.Close() } - if s.Transactions.Idx != nil { - s.Transactions.Idx.Close() + if s.TxnHashIdx != nil { + s.TxnHashIdx.Close() } - if s.Transactions.Segment != nil { - s.Transactions.Segment.Close() + if s.Transactions != nil { + s.Transactions.Close() } } } @@ -304,13 +294,13 @@ func (s *AllSnapshots) Blocks(blockNumber uint64) (snapshot *BlocksSnapshot, fou func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) error { for _, sn := range s.blocks { - f := path.Join(s.dir, SegmentFileName(sn.Headers.From, sn.Headers.To, Headers)) - if err := HeadersHashIdx(f, sn.Headers.From); err != nil { + f := path.Join(s.dir, SegmentFileName(sn.From, sn.To, Headers)) + if err := HeadersHashIdx(f, sn.From); err != nil { return err } - f = path.Join(s.dir, SegmentFileName(sn.Bodies.From, sn.Bodies.To, Bodies)) - if err := BodiesIdx(f, sn.Bodies.From); err != nil { + f = path.Join(s.dir, SegmentFileName(sn.From, sn.To, Bodies)) + if err := BodiesIdx(f, sn.From); err != nil { return err } } @@ -321,7 +311,7 @@ func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) er } for _, sn := range s.blocks { - gg := sn.Bodies.Segment.MakeGetter() + gg := sn.Bodies.MakeGetter() buf, _ := gg.Next(nil) firstBody := &types.BodyForStorage{} if err := rlp.DecodeBytes(buf, firstBody); err != nil { @@ -330,7 +320,7 @@ func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) er var expectedTxsAmount uint64 { - off := sn.Bodies.Idx.Lookup2(sn.To - 1 - sn.From) + off := sn.BodyNumberIdx.Lookup2(sn.To - 1 - sn.From) gg.Reset(off) buf, _ = gg.Next(buf[:0]) @@ -341,7 +331,7 @@ func (s *AllSnapshots) BuildIndices(ctx context.Context, chainID uint256.Int) er } expectedTxsAmount = lastBody.BaseTxId + uint64(lastBody.TxAmount) - firstBody.BaseTxId } - f := path.Join(s.dir, SegmentFileName(sn.Transactions.From, sn.Transactions.To, Transactions)) + f := path.Join(s.dir, SegmentFileName(sn.From, sn.To, Transactions)) if err := TransactionsHashIdx(chainID, firstBody.BaseTxId, f, expectedTxsAmount); err != nil { return err } @@ -756,15 +746,10 @@ func HeadersHashIdx(segmentFileName string, firstBlockNumInSegment uint64) error if err := rlp.DecodeBytes(word, &h); err != nil { return err } - if err := idx.AddKey(h.Hash().Bytes(), offset); err != nil { return err } - // hashBuf := make([]byte, 32) - // common.HashTo(word, hashBuf) - //if err := idx.AddKey(types.RawRlpHash(word).Bytes(), offset); err != nil { - // return err - //} + //TODO: optimize by - types.RawRlpHash(word).Bytes() select { default: @@ -791,11 +776,11 @@ func BodiesIdx(segmentFileName string, firstBlockNumInSegment uint64) error { select { default: case <-logEvery.C: - log.Info("[Snapshots] BodiesIdx", "millions", i/1_000_000) + log.Info("[Snapshots] BodyNumberIdx", "millions", i/1_000_000) } return nil }); err != nil { - return fmt.Errorf("BodiesIdx: %w", err) + return fmt.Errorf("BodyNumberIdx: %w", err) } return nil } @@ -848,7 +833,7 @@ RETRY: if err = rs.Build(); err != nil { if errors.Is(err, recsplit.ErrCollision) { - log.Info("Building recsplit. Collision happened. It's ok. Restarting...", "err", err) + log.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err) rs.ResetNextSalt() goto RETRY } @@ -860,7 +845,7 @@ RETRY: func ForEachHeader(s *AllSnapshots, walker func(header *types.Header) error) error { for _, sn := range s.blocks { - d := sn.Headers.Segment + d := sn.Headers g := d.MakeGetter() word := make([]byte, 0, 4096) header := new(types.Header) @@ -939,22 +924,3 @@ func ReadSimpleFile(fileName string, walker func(v []byte) error) error { } return nil } - -type SnapshotInfo struct { - MinimumAvailability uint64 -} - -func FileCheckSum(file string) (common.Hash, error) { - f, err := os.Open(file) - if err != nil { - return common.Hash{}, err - } - defer f.Close() - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return common.Hash{}, err - } - - hash := h.Sum(nil) - return common.BytesToHash(hash), nil -}