Skip to content

Commit da16d08

Browse files
rjl493456442rjl493456442
authored
trie, les, tests, core: implement trie tracer (#24403)
Trie tracer is an auxiliary tool to capture all deleted nodes which can't be captured by trie.Committer. The deleted nodes can be removed from the disk later.
1 parent 127dc59 commit da16d08

File tree

13 files changed

+413
-35
lines changed

13 files changed

+413
-35
lines changed

‎core/types/hashing_test.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626

2727
"github.com/ethereum/go-ethereum/common"
2828
"github.com/ethereum/go-ethereum/common/hexutil"
29+
"github.com/ethereum/go-ethereum/core/rawdb"
2930
"github.com/ethereum/go-ethereum/core/types"
3031
"github.com/ethereum/go-ethereum/crypto"
3132
"github.com/ethereum/go-ethereum/rlp"
@@ -38,7 +39,8 @@ func TestDeriveSha(t *testing.T) {
3839
t.Fatal(err)
3940
}
4041
for len(txs) < 1000 {
41-
exp := types.DeriveSha(txs, new(trie.Trie))
42+
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
43+
exp := types.DeriveSha(txs, tr)
4244
got := types.DeriveSha(txs, trie.NewStackTrie(nil))
4345
if !bytes.Equal(got[:], exp[:]) {
4446
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@@ -85,7 +87,8 @@ func BenchmarkDeriveSha200(b *testing.B) {
8587
b.ResetTimer()
8688
b.ReportAllocs()
8789
for i := 0; i < b.N; i++ {
88-
exp = types.DeriveSha(txs, new(trie.Trie))
90+
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
91+
exp = types.DeriveSha(txs, tr)
8992
}
9093
})
9194

@@ -106,7 +109,8 @@ func TestFuzzDeriveSha(t *testing.T) {
106109
rndSeed := mrand.Int()
107110
for i := 0; i < 10; i++ {
108111
seed := rndSeed + i
109-
exp := types.DeriveSha(newDummy(i), new(trie.Trie))
112+
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
113+
exp := types.DeriveSha(newDummy(i), tr)
110114
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
111115
if !bytes.Equal(got[:], exp[:]) {
112116
printList(newDummy(seed))
@@ -134,7 +138,8 @@ func TestDerivableList(t *testing.T) {
134138
},
135139
}
136140
for i, tc := range tcs[1:] {
137-
exp := types.DeriveSha(flatList(tc), new(trie.Trie))
141+
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
142+
exp := types.DeriveSha(flatList(tc), tr)
138143
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
139144
if !bytes.Equal(got[:], exp[:]) {
140145
t.Fatalf("case %d: got %x exp %x", i, got, exp)

‎les/server_handler.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccou
374374
return acc, nil
375375
}
376376

377-
// getHelperTrie returns the post-processed trie root for the given trie ID and section index
377+
// GetHelperTrie returns the post-processed trie root for the given trie ID and section index
378378
func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
379379
var (
380380
root common.Hash

‎tests/fuzzers/rangeproof/rangeproof-fuzzer.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sort"
2525

2626
"github.com/ethereum/go-ethereum/common"
27+
"github.com/ethereum/go-ethereum/core/rawdb"
2728
"github.com/ethereum/go-ethereum/ethdb/memorydb"
2829
"github.com/ethereum/go-ethereum/trie"
2930
)
@@ -61,8 +62,7 @@ func (f *fuzzer) readInt() uint64 {
6162
}
6263

6364
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
64-
65-
trie := new(trie.Trie)
65+
trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
6666
vals := make(map[string]*kv)
6767
size := f.readInt()
6868
// Fill it with some fluff

‎trie/committer.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ func (c *committer) commit(n node, db *Database) (node, int, error) {
8989
if hash != nil && !dirty {
9090
return hash, 0, nil
9191
}
92-
// Commit children, then parent, and remove remove the dirty flag.
92+
// Commit children, then parent, and remove the dirty flag.
9393
switch cn := n.(type) {
9494
case *shortNode:
9595
// Commit child

‎trie/iterator_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"testing"
2525

2626
"github.com/ethereum/go-ethereum/common"
27+
"github.com/ethereum/go-ethereum/core/rawdb"
2728
"github.com/ethereum/go-ethereum/crypto"
2829
"github.com/ethereum/go-ethereum/ethdb"
2930
"github.com/ethereum/go-ethereum/ethdb/memorydb"
@@ -296,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
296297
}
297298

298299
func TestIteratorNoDups(t *testing.T) {
299-
var tr Trie
300+
tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
300301
for _, val := range testdata1 {
301302
tr.Update([]byte(val.k), []byte(val.v))
302303
}

‎trie/proof.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323

2424
"github.com/ethereum/go-ethereum/common"
2525
"github.com/ethereum/go-ethereum/ethdb"
26-
"github.com/ethereum/go-ethereum/ethdb/memorydb"
2726
"github.com/ethereum/go-ethereum/log"
2827
)
2928

@@ -552,7 +551,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key
552551
}
553552
// Rebuild the trie with the leaf stream, the shape of trie
554553
// should be same with the original one.
555-
tr := &Trie{root: root, db: NewDatabase(memorydb.New())}
554+
tr := newWithRootNode(root)
556555
if empty {
557556
tr.root = nil
558557
}

‎trie/proof_test.go

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"time"
2727

2828
"github.com/ethereum/go-ethereum/common"
29+
"github.com/ethereum/go-ethereum/core/rawdb"
2930
"github.com/ethereum/go-ethereum/crypto"
3031
"github.com/ethereum/go-ethereum/ethdb/memorydb"
3132
)
@@ -79,7 +80,7 @@ func TestProof(t *testing.T) {
7980
}
8081

8182
func TestOneElementProof(t *testing.T) {
82-
trie := new(Trie)
83+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
8384
updateString(trie, "k", "v")
8485
for i, prover := range makeProvers(trie) {
8586
proof := prover([]byte("k"))
@@ -130,7 +131,7 @@ func TestBadProof(t *testing.T) {
130131
// Tests that missing keys can also be proven. The test explicitly uses a single
131132
// entry trie and checks for missing keys both before and after the single entry.
132133
func TestMissingKeyProof(t *testing.T) {
133-
trie := new(Trie)
134+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
134135
updateString(trie, "k", "v")
135136

136137
for i, key := range []string{"a", "j", "l", "z"} {
@@ -386,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) {
386387
}
387388

388389
// Test the mini trie with only a single element.
389-
tinyTrie := new(Trie)
390+
tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
390391
entry := &kv{randBytes(32), randBytes(20), false}
391392
tinyTrie.Update(entry.k, entry.v)
392393

@@ -458,7 +459,7 @@ func TestAllElementsProof(t *testing.T) {
458459
// TestSingleSideRangeProof tests the range starts from zero.
459460
func TestSingleSideRangeProof(t *testing.T) {
460461
for i := 0; i < 64; i++ {
461-
trie := new(Trie)
462+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
462463
var entries entrySlice
463464
for i := 0; i < 4096; i++ {
464465
value := &kv{randBytes(32), randBytes(20), false}
@@ -493,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) {
493494
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
494495
func TestReverseSingleSideRangeProof(t *testing.T) {
495496
for i := 0; i < 64; i++ {
496-
trie := new(Trie)
497+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
497498
var entries entrySlice
498499
for i := 0; i < 4096; i++ {
499500
value := &kv{randBytes(32), randBytes(20), false}
@@ -600,7 +601,7 @@ func TestBadRangeProof(t *testing.T) {
600601
// TestGappedRangeProof focuses on the small trie with embedded nodes.
601602
// If the gapped node is embedded in the trie, it should be detected too.
602603
func TestGappedRangeProof(t *testing.T) {
603-
trie := new(Trie)
604+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
604605
var entries []*kv // Sorted entries
605606
for i := byte(0); i < 10; i++ {
606607
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@@ -674,7 +675,7 @@ func TestSameSideProofs(t *testing.T) {
674675
}
675676

676677
func TestHasRightElement(t *testing.T) {
677-
trie := new(Trie)
678+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
678679
var entries entrySlice
679680
for i := 0; i < 4096; i++ {
680681
value := &kv{randBytes(32), randBytes(20), false}
@@ -1027,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
10271028
}
10281029

10291030
func randomTrie(n int) (*Trie, map[string]*kv) {
1030-
trie := new(Trie)
1031+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
10311032
vals := make(map[string]*kv)
10321033
for i := byte(0); i < 100; i++ {
10331034
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@@ -1052,7 +1053,7 @@ func randBytes(n int) []byte {
10521053
}
10531054

10541055
func nonRandomTrie(n int) (*Trie, map[string]*kv) {
1055-
trie := new(Trie)
1056+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
10561057
vals := make(map[string]*kv)
10571058
max := uint64(0xffffffffffffffff)
10581059
for i := uint64(0); i < uint64(n); i++ {
@@ -1077,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
10771078
common.Hex2Bytes("02"),
10781079
common.Hex2Bytes("03"),
10791080
}
1080-
trie := new(Trie)
1081+
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
10811082
for i, key := range keys {
10821083
trie.Update(key, vals[i])
10831084
}

‎trie/secure_trie.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) {
8787
return t.trie.TryGetNode(path)
8888
}
8989

90-
// TryUpdate account will abstract the write of an account to the
90+
// TryUpdateAccount account will abstract the write of an account to the
9191
// secure trie.
9292
func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error {
9393
hk := t.hashKey(key)
@@ -185,8 +185,10 @@ func (t *SecureTrie) Hash() common.Hash {
185185

186186
// Copy returns a copy of SecureTrie.
187187
func (t *SecureTrie) Copy() *SecureTrie {
188-
cpy := *t
189-
return &cpy
188+
return &SecureTrie{
189+
trie: *t.trie.Copy(),
190+
secKeyCache: t.secKeyCache,
191+
}
190192
}
191193

192194
// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration

‎trie/secure_trie_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
112112
threads := runtime.NumCPU()
113113
tries := make([]*SecureTrie, threads)
114114
for i := 0; i < threads; i++ {
115-
cpy := *trie
116-
tries[i] = &cpy
115+
tries[i] = trie.Copy()
117116
}
118117
// Start a batch of goroutines interactng with the trie
119118
pend := new(sync.WaitGroup)

‎trie/trie.go

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sync"
2525

2626
"github.com/ethereum/go-ethereum/common"
27+
"github.com/ethereum/go-ethereum/core/rawdb"
2728
"github.com/ethereum/go-ethereum/core/types"
2829
"github.com/ethereum/go-ethereum/crypto"
2930
"github.com/ethereum/go-ethereum/log"
@@ -62,17 +63,32 @@ type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent commo
6263
type Trie struct {
6364
db *Database
6465
root node
65-
// Keep track of the number leafs which have been inserted since the last
66+
67+
// Keep track of the number leaves which have been inserted since the last
6668
// hashing operation. This number will not directly map to the number of
6769
// actually unhashed nodes
6870
unhashed int
71+
72+
// tracer is the state diff tracer can be used to track newly added/deleted
73+
// trie node. It will be reset after each commit operation.
74+
tracer *tracer
6975
}
7076

7177
// newFlag returns the cache flag value for a newly created node.
7278
func (t *Trie) newFlag() nodeFlag {
7379
return nodeFlag{dirty: true}
7480
}
7581

82+
// Copy returns a copy of Trie.
83+
func (t *Trie) Copy() *Trie {
84+
return &Trie{
85+
db: t.db,
86+
root: t.root,
87+
unhashed: t.unhashed,
88+
tracer: t.tracer.copy(),
89+
}
90+
}
91+
7692
// New creates a trie with an existing root node from db.
7793
//
7894
// If root is the zero hash or the sha3 hash of an empty string, the
@@ -85,6 +101,7 @@ func New(root common.Hash, db *Database) (*Trie, error) {
85101
}
86102
trie := &Trie{
87103
db: db,
104+
//tracer: newTracer(),
88105
}
89106
if root != (common.Hash{}) && root != emptyRoot {
90107
rootnode, err := trie.resolveHash(root[:], nil)
@@ -96,6 +113,16 @@ func New(root common.Hash, db *Database) (*Trie, error) {
96113
return trie, nil
97114
}
98115

116+
// newWithRootNode initializes the trie with the given root node.
117+
// It's only used by range prover.
118+
func newWithRootNode(root node) *Trie {
119+
return &Trie{
120+
root: root,
121+
//tracer: newTracer(),
122+
db: NewDatabase(rawdb.NewMemoryDatabase()),
123+
}
124+
}
125+
99126
// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
100127
// the key after the given start key.
101128
func (t *Trie) NodeIterator(start []byte) NodeIterator {
@@ -317,7 +344,12 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
317344
if matchlen == 0 {
318345
return true, branch, nil
319346
}
320-
// Otherwise, replace it with a short node leading up to the branch.
347+
// New branch node is created as a child of the original short node.
348+
// Track the newly inserted node in the tracer. The node identifier
349+
// passed is the path from the root node.
350+
t.tracer.onInsert(append(prefix, key[:matchlen]...))
351+
352+
// Replace it with a short node leading up to the branch.
321353
return true, &shortNode{key[:matchlen], branch, t.newFlag()}, nil
322354

323355
case *fullNode:
@@ -331,6 +363,11 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
331363
return true, n, nil
332364

333365
case nil:
366+
// New short node is created and track it in the tracer. The node identifier
367+
// passed is the path from the root node. Note the valueNode won't be tracked
368+
// since it's always embedded in its parent.
369+
t.tracer.onInsert(prefix)
370+
334371
return true, &shortNode{key, value, t.newFlag()}, nil
335372

336373
case hashNode:
@@ -383,6 +420,11 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
383420
return false, n, nil // don't replace n on mismatch
384421
}
385422
if matchlen == len(key) {
423+
// The matched short node is deleted entirely and track
424+
// it in the deletion set. The same the valueNode doesn't
425+
// need to be tracked at all since it's always embedded.
426+
t.tracer.onDelete(prefix)
427+
386428
return true, nil, nil // remove n entirely for whole matches
387429
}
388430
// The key is longer than n.Key. Remove the remaining suffix
@@ -395,6 +437,10 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
395437
}
396438
switch child := child.(type) {
397439
case *shortNode:
440+
// The child shortNode is merged into its parent, track
441+
// is deleted as well.
442+
t.tracer.onDelete(append(prefix, n.Key...))
443+
398444
// Deleting from the subtrie reduced it to another
399445
// short node. Merge the nodes to avoid creating a
400446
// shortNode{..., shortNode{...}}. Use concat (which
@@ -456,6 +502,11 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
456502
return false, nil, err
457503
}
458504
if cnode, ok := cnode.(*shortNode); ok {
505+
// Replace the entire full node with the short node.
506+
// Mark the original short node as deleted since the
507+
// value is embedded into the parent now.
508+
t.tracer.onDelete(append(prefix, byte(pos)))
509+
459510
k := append([]byte{byte(pos)}, cnode.Key...)
460511
return true, &shortNode{k, cnode.Val, t.newFlag()}, nil
461512
}
@@ -537,6 +588,8 @@ func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
537588
if t.db == nil {
538589
panic("commit called on trie with nil database")
539590
}
591+
defer t.tracer.reset()
592+
540593
if t.root == nil {
541594
return emptyRoot, 0, nil
542595
}
@@ -595,4 +648,5 @@ func (t *Trie) hashRoot() (node, node, error) {
595648
func (t *Trie) Reset() {
596649
t.root = nil
597650
t.unhashed = 0
651+
t.tracer.reset()
598652
}

0 commit comments

Comments
 (0)