core: remove block numbers from snapshots to make a pure state tree

Author: karalabe

core/blockchain.go

@@ -301,7 +301,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	}
 	// Load any existing snapshot, regenerating it if loading failed
 	head := bc.CurrentBlock()
-	if bc.snaps, err = snapshot.New(bc.db, "snapshot.rlp", head.NumberU64(), head.Root()); err != nil {
+	if bc.snaps, err = snapshot.New(bc.db, "snapshot.rlp", head.Root()); err != nil {
 		return nil, err
 	}
 	// Take ownership of this particular state

core/rawdb/accessors_snapshot.go

@@ -17,38 +17,36 @@
 package rawdb
 
 import (
-	"encoding/binary"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 )
 
-// ReadSnapshotBlock retrieves the number and root of the block whose state is
-// contained in the persisted snapshot.
-func ReadSnapshotBlock(db ethdb.KeyValueReader) (uint64, common.Hash) {
-	data, _ := db.Get(snapshotBlockKey)
-	if len(data) != 8+common.HashLength {
-		return 0, common.Hash{}
+// ReadSnapshotRoot retrieves the root of the block whose state is contained in
+// the persisted snapshot.
+func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash {
+	data, _ := db.Get(snapshotRootKey)
+	if len(data) != common.HashLength {
+		return common.Hash{}
 	}
-	return binary.BigEndian.Uint64(data[:8]), common.BytesToHash(data[8:])
+	return common.BytesToHash(data)
 }
 
-// WriteSnapshotBlock stores the number and root of the block whose state is
-// contained in the persisted snapshot.
-func WriteSnapshotBlock(db ethdb.KeyValueWriter, number uint64, root common.Hash) {
-	if err := db.Put(snapshotBlockKey, append(encodeBlockNumber(number), root.Bytes()...)); err != nil {
-		log.Crit("Failed to store snapsnot block's number and root", "err", err)
+// WriteSnapshotRoot stores the root of the block whose state is contained in
+// the persisted snapshot.
+func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) {
+	if err := db.Put(snapshotRootKey, root[:]); err != nil {
+		log.Crit("Failed to store snapshot root", "err", err)
 	}
 }
 
-// DeleteSnapshotBlock deletes the number and hash of the block whose state is
-// contained in the persisted snapshot. Since snapshots are not immutable, this
-// method can be used during updates, so a crash or failure will mark the entire
-// snapshot invalid.
-func DeleteSnapshotBlock(db ethdb.KeyValueWriter) {
-	if err := db.Delete(snapshotBlockKey); err != nil {
-		log.Crit("Failed to remove snapsnot block's number and hash", "err", err)
+// DeleteSnapshotRoot deletes the hash of the block whose state is contained in
+// the persisted snapshot. Since snapshots are not immutable, this  method can
+// be used during updates, so a crash or failure will mark the entire snapshot
+// invalid.
+func DeleteSnapshotRoot(db ethdb.KeyValueWriter) {
+	if err := db.Delete(snapshotRootKey); err != nil {
+		log.Crit("Failed to remove snapshot root", "err", err)
 	}
 }
 

core/rawdb/schema.go

@@ -41,8 +41,8 @@ var (
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
 	fastTrieProgressKey = []byte("TrieSync")
 
-	// snapshotBlockKey tracks the number and hash of the last snapshot.
-	snapshotBlockKey = []byte("SnapshotBlock")
+	// snapshotRootKey tracks the number and hash of the last snapshot.
+	snapshotRootKey = []byte("SnapshotRoot")
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header

core/state/snapshot/difflayer.go

@@ -36,9 +36,8 @@ type diffLayer struct {
 	parent snapshot // Parent snapshot modified by this one, never nil
 	memory uint64   // Approximate guess as to how much memory we use
 
-	number uint64      // Block number to which this snapshot diff belongs to
-	root   common.Hash // Root hash to which this snapshot diff belongs to
-	stale  bool        // Signals that the layer became stale (state progressed)
+	root  common.Hash // Root hash to which this snapshot diff belongs to
+	stale bool        // Signals that the layer became stale (state progressed)
 
 	accountList []common.Hash                          // List of account for iteration. If it exists, it's sorted, otherwise it's nil
 	accountData map[common.Hash][]byte                 // Keyed accounts for direct retrival (nil means deleted)
@@ -50,11 +49,10 @@ type diffLayer struct {
 
 // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
 // level persistent database or a hierarchical diff already.
-func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
+func newDiffLayer(parent snapshot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
 	// Create the new layer with some pre-allocated data segments
 	dl := &diffLayer{
 		parent:      parent,
-		number:      number,
 		root:        root,
 		accountData: accounts,
 		storageData: storage,
@@ -63,7 +61,6 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map
 	for _, data := range accounts {
 		dl.memory += uint64(len(data))
 	}
-
 	// Fill the storage hashes and sort them for the iterator
 	dl.storageList = make(map[common.Hash][]common.Hash)
 
@@ -93,9 +90,27 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map
 	return dl
 }
 
-// Info returns the block number and root hash for which this snapshot was made.
-func (dl *diffLayer) Info() (uint64, common.Hash) {
-	return dl.number, dl.root
+// Root returns the root hash for which this snapshot was made.
+func (dl *diffLayer) Root() common.Hash {
+	return dl.root
+}
+
+// Stale return whether this layer has become stale (was flattened across) or if
+// it's still live.
+func (dl *diffLayer) Stale() bool {
+	// If the parent is stale, mark this layer as stale too
+	if stale := dl.parent.Stale(); stale {
+		dl.lock.Lock()
+		defer dl.lock.Unlock()
+
+		dl.stale = true
+		return true
+	}
+	// Parent not stale, return only whether we are
+	dl.lock.RLock()
+	defer dl.lock.RUnlock()
+
+	return dl.stale
 }
 
 // Account directly retrieves the account associated with a particular hash in
@@ -164,7 +179,7 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
 // Update creates a new layer on top of the existing snapshot diff tree with
 // the specified data items.
 func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
-	return newDiffLayer(dl, dl.number+1, blockRoot, accounts, storage)
+	return newDiffLayer(dl, blockRoot, accounts, storage)
 }
 
 // flatten pushes all data from this point downwards, flattening everything into
@@ -213,7 +228,6 @@ func (dl *diffLayer) flatten() snapshot {
 	// Return the combo parent
 	return &diffLayer{
 		parent:      parent.parent,
-		number:      dl.number,
 		root:        dl.root,
 		storageList: parent.storageList,
 		storageData: parent.storageData,

core/state/snapshot/difflayer_journal.go

@@ -43,18 +43,12 @@ type journalStorage struct {
 // diff and verifying that it can be linked to the requested parent.
 func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
 	// Read the next diff journal entry
-	var (
-		number uint64
-		root   common.Hash
-	)
-	if err := r.Decode(&number); err != nil {
+	var root common.Hash
+	if err := r.Decode(&root); err != nil {
 		// The first read may fail with EOF, marking the end of the journal
 		if err == io.EOF {
 			return parent, nil
 		}
-		return nil, fmt.Errorf("load diff number: %v", err)
-	}
-	if err := r.Decode(&root); err != nil {
 		return nil, fmt.Errorf("load diff root: %v", err)
 	}
 	var accounts []journalAccount
@@ -77,13 +71,7 @@ func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
 		}
 		storageData[entry.Hash] = slots
 	}
-	// Validate the block number to avoid state corruption
-	if parent, ok := parent.(*diffLayer); ok {
-		if number != parent.number+1 {
-			return nil, fmt.Errorf("snapshot chain broken: block #%d after #%d", number, parent.number)
-		}
-	}
-	return loadDiffLayer(newDiffLayer(parent, number, root, accountData, storageData), r)
+	return loadDiffLayer(newDiffLayer(parent, root, accountData, storageData), r)
 }
 
 // journal is the internal version of Journal that also returns the journal file
@@ -113,13 +101,8 @@ func (dl *diffLayer) journal() (io.WriteCloser, error) {
 		writer.Close()
 		return nil, ErrSnapshotStale
 	}
-	buf := bufio.NewWriter(writer)
 	// Everything below was journalled, persist this layer too
-	if err := rlp.Encode(buf, dl.number); err != nil {
-		buf.Flush()
-		writer.Close()
-		return nil, err
-	}
+	buf := bufio.NewWriter(writer)
 	if err := rlp.Encode(buf, dl.root); err != nil {
 		buf.Flush()
 		writer.Close()

core/state/snapshot/difflayer_test.go

@@ -61,11 +61,11 @@ func TestMergeBasics(t *testing.T) {
 		}
 	}
 	// Add some (identical) layers on top
-	parent := newDiffLayer(emptyLayer{}, 1, common.Hash{}, accounts, storage)
-	child := newDiffLayer(parent, 1, common.Hash{}, accounts, storage)
-	child = newDiffLayer(child, 1, common.Hash{}, accounts, storage)
-	child = newDiffLayer(child, 1, common.Hash{}, accounts, storage)
-	child = newDiffLayer(child, 1, common.Hash{}, accounts, storage)
+	parent := newDiffLayer(emptyLayer{}, common.Hash{}, accounts, storage)
+	child := newDiffLayer(parent, common.Hash{}, accounts, storage)
+	child = newDiffLayer(child, common.Hash{}, accounts, storage)
+	child = newDiffLayer(child, common.Hash{}, accounts, storage)
+	child = newDiffLayer(child, common.Hash{}, accounts, storage)
 	// And flatten
 	merged := (child.flatten()).(*diffLayer)
 
@@ -122,7 +122,7 @@ func TestMergeDelete(t *testing.T) {
 	}
 
 	// Add some flip-flopping layers on top
-	parent := newDiffLayer(emptyLayer{}, 1, common.Hash{}, flip(), storage)
+	parent := newDiffLayer(emptyLayer{}, common.Hash{}, flip(), storage)
 	child := parent.Update(common.Hash{}, flop(), storage)
 	child = child.Update(common.Hash{}, flip(), storage)
 	child = child.Update(common.Hash{}, flop(), storage)
@@ -139,10 +139,6 @@ func TestMergeDelete(t *testing.T) {
 	// And flatten
 	merged := (child.flatten()).(*diffLayer)
 
-	// check number
-	if got, exp := merged.number, child.number; got != exp {
-		t.Errorf("merged layer: wrong number - exp %d got %d", exp, got)
-	}
 	if data, _ := merged.Account(h1); data == nil {
 		t.Errorf("merged layer: expected %x to be non-nil", h1)
 	}
@@ -169,7 +165,7 @@ func TestInsertAndMerge(t *testing.T) {
 	{
 		var accounts = make(map[common.Hash][]byte)
 		var storage = make(map[common.Hash]map[common.Hash][]byte)
-		parent = newDiffLayer(emptyLayer{}, 1, common.Hash{}, accounts, storage)
+		parent = newDiffLayer(emptyLayer{}, common.Hash{}, accounts, storage)
 	}
 	{
 		var accounts = make(map[common.Hash][]byte)
@@ -178,7 +174,7 @@ func TestInsertAndMerge(t *testing.T) {
 		accstorage := make(map[common.Hash][]byte)
 		storage[acc] = accstorage
 		storage[acc][slot] = []byte{0x01}
-		child = newDiffLayer(parent, 2, common.Hash{}, accounts, storage)
+		child = newDiffLayer(parent, common.Hash{}, accounts, storage)
 	}
 	// And flatten
 	merged := (child.flatten()).(*diffLayer)
@@ -200,11 +196,12 @@ func (emptyLayer) Journal() error {
 	panic("implement me")
 }
 
-func (emptyLayer) Info() (uint64, common.Hash) {
-	return 0, common.Hash{}
+func (emptyLayer) Stale() bool {
+	panic("implement me")
 }
-func (emptyLayer) Number() uint64 {
-	return 0
+
+func (emptyLayer) Root() common.Hash {
+	return common.Hash{}
 }
 
 func (emptyLayer) Account(hash common.Hash) (*Account, error) {
@@ -227,19 +224,15 @@ func (emptyLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error)
 // BenchmarkSearch-6   	  500000	      3723 ns/op (10k per layer, only top-level RLock()
 func BenchmarkSearch(b *testing.B) {
 	// First, we set up 128 diff layers, with 1K items each
-
-	blocknum := uint64(0)
 	fill := func(parent snapshot) *diffLayer {
 		accounts := make(map[common.Hash][]byte)
 		storage := make(map[common.Hash]map[common.Hash][]byte)
 
 		for i := 0; i < 10000; i++ {
 			accounts[randomHash()] = randomAccount()
 		}
-		blocknum++
-		return newDiffLayer(parent, blocknum, common.Hash{}, accounts, storage)
+		return newDiffLayer(parent, common.Hash{}, accounts, storage)
 	}
-
 	var layer snapshot
 	layer = emptyLayer{}
 	for i := 0; i < 128; i++ {
@@ -261,8 +254,6 @@ func BenchmarkSearch(b *testing.B) {
 // BenchmarkSearchSlot-6   	  100000	     14551 ns/op (when checking parent number using atomic)
 func BenchmarkSearchSlot(b *testing.B) {
 	// First, we set up 128 diff layers, with 1K items each
-
-	blocknum := uint64(0)
 	accountKey := common.Hash{}
 	storageKey := common.HexToHash("0x1337")
 	accountRLP := randomAccount()
@@ -278,16 +269,13 @@ func BenchmarkSearchSlot(b *testing.B) {
 			accStorage[randomHash()] = value
 			storage[accountKey] = accStorage
 		}
-		blocknum++
-		return newDiffLayer(parent, blocknum, common.Hash{}, accounts, storage)
+		return newDiffLayer(parent, common.Hash{}, accounts, storage)
 	}
-
 	var layer snapshot
 	layer = emptyLayer{}
 	for i := 0; i < 128; i++ {
 		layer = fill(layer)
 	}
-
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		layer.Storage(accountKey, storageKey)
@@ -300,7 +288,7 @@ func BenchmarkSearchSlot(b *testing.B) {
 // Without sorting and tracking accountlist
 // BenchmarkFlatten-6   	     300	   5511511 ns/op
 func BenchmarkFlatten(b *testing.B) {
-	fill := func(parent snapshot, blocknum int) *diffLayer {
+	fill := func(parent snapshot) *diffLayer {
 		accounts := make(map[common.Hash][]byte)
 		storage := make(map[common.Hash]map[common.Hash][]byte)
 
@@ -317,7 +305,7 @@ func BenchmarkFlatten(b *testing.B) {
 			}
 			storage[accountKey] = accStorage
 		}
-		return newDiffLayer(parent, uint64(blocknum), common.Hash{}, accounts, storage)
+		return newDiffLayer(parent, common.Hash{}, accounts, storage)
 	}
 
 	b.ResetTimer()
@@ -327,7 +315,7 @@ func BenchmarkFlatten(b *testing.B) {
 		var layer snapshot
 		layer = emptyLayer{}
 		for i := 1; i < 128; i++ {
-			layer = fill(layer, i)
+			layer = fill(layer)
 		}
 		b.StartTimer()
 
@@ -336,7 +324,6 @@ func BenchmarkFlatten(b *testing.B) {
 			if !ok {
 				break
 			}
-
 			layer = dl.flatten()
 		}
 		b.StopTimer()
@@ -351,7 +338,7 @@ func BenchmarkFlatten(b *testing.B) {
 // BenchmarkJournal-6   	       1	1471373923 ns/ops
 // BenchmarkJournal-6   	       1	1208083335 ns/op // bufio writer
 func BenchmarkJournal(b *testing.B) {
-	fill := func(parent snapshot, blocknum int) *diffLayer {
+	fill := func(parent snapshot) *diffLayer {
 		accounts := make(map[common.Hash][]byte)
 		storage := make(map[common.Hash]map[common.Hash][]byte)
 
@@ -368,15 +355,14 @@ func BenchmarkJournal(b *testing.B) {
 			}
 			storage[accountKey] = accStorage
 		}
-		return newDiffLayer(parent, uint64(blocknum), common.Hash{}, accounts, storage)
+		return newDiffLayer(parent, common.Hash{}, accounts, storage)
 	}
-
 	var layer snapshot
 	layer = &diskLayer{
 		journal: path.Join(os.TempDir(), "difflayer_journal.tmp"),
 	}
 	for i := 1; i < 128; i++ {
-		layer = fill(layer, i)
+		layer = fill(layer)
 	}
 	b.ResetTimer()