Body snapshot (erigontech#2100)

* fix test * get rid of ObjectDatabase * sn_builder_prototype2 * save state * save state * integration step1 * fix lint * fix * fix test * integrate migrator.finish * fix lint * fix build * fix typo * save state * body snapshot test * unique tx * body snapshot generation using walk * move methods out of test * block data verification added * fix lint * test with remove works correctly * fix lint * remove experiment test * fix test * add comment * add second layer of remove test * rename test * fix typos * fix lint * revert testdata * body snapshot migration save state * fix body snapshot migration * fix after merge * remove debug test * debug windows build * fix build * fix * fix lint * debug * fix * fix windows build * simplify snapshot management&&get rid of lazy tx * fix lint * fix windows path * debug * debug * debug * debug * remove geometry experiments * skip windows tests * clean * fix * fix ;int
0xPolygonHermez · Jul 6, 2021 · 5747317 · 5747317
1 parent ea5abbe
commit 5747317
Show file tree

Hide file tree

Showing 19 changed files with 1,546 additions and 429 deletions.
diff --git a/cmd/integration/commands/snapshot_check.go b/cmd/integration/commands/snapshot_check.go
@@ -81,9 +81,12 @@ var cmdSnapshotCheck = &cobra.Command{
 		tmpDb := kv2.NewMDBX().Path(path).MustOpen()
 		kv := kv2.NewSnapshotKV().
 			DB(tmpDb).
-			SnapshotDB([]string{dbutils.HeadersBucket, dbutils.HeaderCanonicalBucket, dbutils.HeaderTDBucket, dbutils.BlockBodyPrefix, dbutils.Senders, dbutils.HeadBlockKey, dbutils.HeaderNumberBucket}, mainDB.RwKV()).
-			SnapshotDB([]string{dbutils.PlainStateBucket, dbutils.CodeBucket, dbutils.PlainContractCodeBucket}, stateSnapshot).
+			//broken
+			//SnapshotDB([]string{dbutils.HeadersBucket, dbutils.HeaderCanonicalBucket, dbutils.HeaderTDBucket, dbutils.BlockBodyPrefix, dbutils.Senders, dbutils.HeadBlockKey, dbutils.HeaderNumberBucket}, mainDB.RwKV()).
+			//SnapshotDB([]string{dbutils.PlainStateBucket, dbutils.CodeBucket, dbutils.PlainContractCodeBucket}, stateSnapshot).
 			Open()
+		_ = mainDB
+		_ = stateSnapshot
 
 		if isNew {
 			if err := kv.Update(ctx, func(tx ethdb.RwTx) error {

diff --git a/cmd/snapshots/debug/debug_test.go b/cmd/snapshots/debug/debug_test.go
@@ -67,7 +67,11 @@ func TestMatreshkaStream(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	snkv := kv2.NewSnapshotKV().DB(kv).SnapshotDB([]string{dbutils.HeadersBucket, dbutils.HeaderCanonicalBucket, dbutils.HeaderTDBucket, dbutils.HeaderNumberBucket, dbutils.BlockBodyPrefix, dbutils.HeadHeaderKey, dbutils.Senders}, chaindata.RwKV()).Open()
+	snkv := kv2.NewSnapshotKV().DB(kv).
+		//broken
+		//SnapshotDB([]string{dbutils.HeadersBucket, dbutils.HeaderCanonicalBucket, dbutils.HeaderTDBucket, dbutils.HeaderNumberBucket, dbutils.BlockBodyPrefix, dbutils.HeadHeaderKey, dbutils.Senders}, chaindata.RwKV()).
+		Open()
+	_ = chaindata
 	defer snkv.Close()
 	db := kv2.NewObjectDatabase(snkv)
 

diff --git a/cmd/snapshots/generator/commands/generate_body_snapshot.go b/cmd/snapshots/generator/commands/generate_body_snapshot.go
@@ -3,7 +3,6 @@ package commands
 import (
 	"context"
 	"fmt"
-	"math/big"
 	"os"
 	"time"
 
@@ -38,8 +37,7 @@ func BodySnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock uint
 	kv := kv2.NewMDBX().Path(dbPath).MustOpen()
 	snKV := kv2.NewMDBX().WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
 		return dbutils.BucketsCfg{
-			dbutils.BlockBodyPrefix:          dbutils.BucketConfigItem{},
-			dbutils.BodiesSnapshotInfoBucket: dbutils.BucketConfigItem{},
+			dbutils.BlockBodyPrefix: dbutils.BucketConfigItem{},
 		}
 	}).Path(snapshotPath).MustOpen()
 
@@ -74,17 +72,6 @@ func BodySnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock uint
 			default:
 			}
 		}
-
-		err = sntx.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadNumber), big.NewInt(0).SetUint64(toBlock).Bytes())
-		if err != nil {
-			log.Crit("SnapshotBodyHeadNumber error", "err", err)
-			return err
-		}
-		err = sntx.Put(dbutils.BodiesSnapshotInfoBucket, []byte(dbutils.SnapshotBodyHeadHash), hash.Bytes())
-		if err != nil {
-			log.Crit("SnapshotBodyHeadHash error", "err", err)
-			return err
-		}
 		return nil
 	}); err != nil {
 		return err

diff --git a/cmd/snapshots/generator/commands/generate_header_snapshot.go b/cmd/snapshots/generator/commands/generate_header_snapshot.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"math/big"
 	"os"
 	"time"
 
@@ -90,16 +89,6 @@ func HeaderSnapshot(ctx context.Context, dbPath, snapshotPath string, toBlock ui
 		}
 	}
 
-	err = snTx.Put(dbutils.HeadersSnapshotInfoBucket, []byte(dbutils.SnapshotHeadersHeadNumber), big.NewInt(0).SetUint64(toBlock).Bytes())
-	if err != nil {
-		log.Crit("SnapshotHeadersHeadNumber error", "err", err)
-		return err
-	}
-	err = snTx.Put(dbutils.HeadersSnapshotInfoBucket, []byte(dbutils.SnapshotHeadersHeadHash), hash.Bytes())
-	if err != nil {
-		log.Crit("SnapshotHeadersHeadHash error", "err", err)
-		return err
-	}
 	if err = snTx.Commit(); err != nil {
 		return err
 	}

diff --git a/cmd/snapshots/generator/commands/generate_state_snapshot.go b/cmd/snapshots/generator/commands/generate_state_snapshot.go
@@ -52,7 +52,6 @@ func GenerateStateSnapshot(ctx context.Context, dbPath, snapshotPath string, toB
 			dbutils.PlainStateBucket:        dbutils.BucketConfigItem{},
 			dbutils.PlainContractCodeBucket: dbutils.BucketConfigItem{},
 			dbutils.CodeBucket:              dbutils.BucketConfigItem{},
-			dbutils.StateSnapshotInfoBucket: dbutils.BucketConfigItem{},
 		}
 	}).Path(snapshotPath).MustOpen()
 

diff --git a/cmd/snapshots/generator/commands/verify_state_snapshot.go b/cmd/snapshots/generator/commands/verify_state_snapshot.go
@@ -51,7 +51,7 @@ func VerifyStateSnapshot(ctx context.Context, dbPath, snapshotPath string, block
 
 	defer os.RemoveAll(tmpPath)
 	defer tmpDB.Close()
-	snkv = kv.NewSnapshotKV().SnapshotDB([]string{dbutils.PlainStateBucket, dbutils.PlainContractCodeBucket, dbutils.CodeBucket}, snkv).DB(tmpDB).Open()
+	snkv = kv.NewSnapshotKV().StateSnapshot(snkv).DB(tmpDB).Open()
 	tx, err := snkv.BeginRw(context.Background())
 	if err != nil {
 		return err

diff --git a/cmd/state/verify/verify_headers_snapshot.go b/cmd/state/verify/verify_headers_snapshot.go
@@ -15,8 +15,7 @@ import (
 func HeadersSnapshot(snapshotPath string) error {
 	snKV := kv.NewMDBX().Path(snapshotPath).Readonly().WithBucketsConfig(func(defaultBuckets dbutils.BucketsCfg) dbutils.BucketsCfg {
 		return dbutils.BucketsCfg{
-			dbutils.HeadersBucket:             dbutils.BucketConfigItem{},
-			dbutils.HeadersSnapshotInfoBucket: dbutils.BucketConfigItem{},
+			dbutils.HeadersBucket: dbutils.BucketConfigItem{},
 		}
 	}).MustOpen()
 	var prevHeader *types.Header

diff --git a/common/dbutils/bucket.go b/common/dbutils/bucket.go
@@ -187,12 +187,9 @@ const TrieOfStorageBucket = "TrieStorage"
 
 const (
 	// DatabaseInfoBucket is used to store information about data layout.
-	DatabaseInfoBucket        = "DbInfo"
-	SnapshotInfoBucket        = "SnapshotInfo"
-	BittorrentInfoBucket      = "BittorrentInfo"
-	HeadersSnapshotInfoBucket = "HeadersSnapshotInfo"
-	BodiesSnapshotInfoBucket  = "BodiesSnapshotInfo"
-	StateSnapshotInfoBucket   = "StateSnapshotInfo"
+	DatabaseInfoBucket   = "DbInfo"
+	SnapshotInfoBucket   = "SnapshotInfo"
+	BittorrentInfoBucket = "BittorrentInfo"
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	HeaderPrefixOld    = "h"            // block_num_u64 + hash -> header
@@ -280,14 +277,11 @@ var (
 
 	DBSchemaVersionKey = []byte("dbVersion")
 
-	SnapshotHeadersHeadNumber = "SnapshotLastHeaderNumber"
-	SnapshotHeadersHeadHash   = "SnapshotLastHeaderHash"
-	SnapshotBodyHeadNumber    = "SnapshotLastBodyNumber"
-	SnapshotBodyHeadHash      = "SnapshotLastBodyHash"
-
 	BittorrentPeerID            = "peerID"
 	CurrentHeadersSnapshotHash  = []byte("CurrentHeadersSnapshotHash")
 	CurrentHeadersSnapshotBlock = []byte("CurrentHeadersSnapshotBlock")
+	CurrentBodiesSnapshotHash   = []byte("CurrentBodiesSnapshotHash")
+	CurrentBodiesSnapshotBlock  = []byte("CurrentBodiesSnapshotBlock")
 )
 
 // Buckets - list of all buckets. App will panic if some bucket is not in this list.
@@ -324,9 +318,6 @@ var Buckets = []string{
 	LogTopicIndex,
 	LogAddressIndex,
 	SnapshotInfoBucket,
-	HeadersSnapshotInfoBucket,
-	BodiesSnapshotInfoBucket,
-	StateSnapshotInfoBucket,
 	CallTraceSet,
 	CallFromIndex,
 	CallToIndex,