Skip to content

Commit 1ee650b

Browse files
committed
all: implement path-based state scheme
1 parent c87f321 commit 1ee650b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+4583
-1379
lines changed

cmd/geth/snapshot.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ block is used.
165165
}
166166
)
167167

168+
// Deprecation: this command should be deprecated once the hash-based
169+
// scheme is deprecated.
168170
func pruneState(ctx *cli.Context) error {
169171
stack, config := makeConfigNode(ctx)
170172
defer stack.Close()
@@ -399,7 +401,7 @@ func traverseRawState(ctx *cli.Context) error {
399401
// Check the present for non-empty hash node(embedded node doesn't
400402
// have their own hash).
401403
if node != (common.Hash{}) {
402-
blob := rawdb.ReadTrieNode(chaindb, node)
404+
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
403405
if len(blob) == 0 {
404406
log.Error("Missing trie node(account)", "hash", node)
405407
return errors.New("missing account")
@@ -433,10 +435,10 @@ func traverseRawState(ctx *cli.Context) error {
433435
nodes += 1
434436
node := storageIter.Hash()
435437

436-
// Check the present for non-empty hash node(embedded node doesn't
438+
// Check the presence for non-empty hash node(embedded node doesn't
437439
// have their own hash).
438440
if node != (common.Hash{}) {
439-
blob := rawdb.ReadTrieNode(chaindb, node)
441+
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
440442
if len(blob) == 0 {
441443
log.Error("Missing trie node(storage)", "hash", node)
442444
return errors.New("missing storage")

core/blockchain.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -913,14 +913,14 @@ func (bc *BlockChain) Stop() {
913913
recent := bc.GetBlockByNumber(number - offset)
914914

915915
log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
916-
if err := triedb.Commit(recent.Root(), true, nil); err != nil {
916+
if err := triedb.Commit(recent.Root(), true); err != nil {
917917
log.Error("Failed to commit recent state trie", "err", err)
918918
}
919919
}
920920
}
921921
if snapBase != (common.Hash{}) {
922922
log.Info("Writing snapshot state to disk", "root", snapBase)
923-
if err := triedb.Commit(snapBase, true, nil); err != nil {
923+
if err := triedb.Commit(snapBase, true); err != nil {
924924
log.Error("Failed to commit recent state trie", "err", err)
925925
}
926926
}
@@ -932,8 +932,8 @@ func (bc *BlockChain) Stop() {
932932
}
933933
}
934934
// Flush the collected preimages to disk
935-
if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil {
936-
log.Error("Failed to commit trie preimages", "err", err)
935+
if err := bc.stateCache.TrieDB().Close(); err != nil {
936+
log.Error("Failed to close trie db", "err", err)
937937
}
938938
// Ensure all live cached entries be saved into disk, so that we can skip
939939
// cache warmup when node restarts.
@@ -1311,7 +1311,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
13111311
}
13121312
// If we're running an archive node, always flush
13131313
if bc.cacheConfig.TrieDirtyDisabled {
1314-
return bc.triedb.Commit(root, false, nil)
1314+
return bc.triedb.Commit(root, false)
13151315
}
13161316
// Full but not archive node, do proper garbage collection
13171317
bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
@@ -1347,7 +1347,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
13471347
log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "optimum", float64(chosen-bc.lastWrite)/TriesInMemory)
13481348
}
13491349
// Flush an entire trie and restart the counters
1350-
bc.triedb.Commit(header.Root, true, nil)
1350+
bc.triedb.Commit(header.Root, true)
13511351
bc.lastWrite = chosen
13521352
bc.gcproc = 0
13531353
}

core/blockchain_repair_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1803,7 +1803,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
18031803
t.Fatalf("Failed to import canonical chain start: %v", err)
18041804
}
18051805
if tt.commitBlock > 0 {
1806-
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
1806+
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
18071807
if snapshots {
18081808
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
18091809
t.Fatalf("Failed to flatten snapshots: %v", err)
@@ -1918,7 +1918,7 @@ func TestIssue23496(t *testing.T) {
19181918
if _, err := chain.InsertChain(blocks[:1]); err != nil {
19191919
t.Fatalf("Failed to import canonical chain start: %v", err)
19201920
}
1921-
chain.stateCache.TrieDB().Commit(blocks[0].Root(), true, nil)
1921+
chain.stateCache.TrieDB().Commit(blocks[0].Root(), false)
19221922

19231923
// Insert block B2 and commit the snapshot into disk
19241924
if _, err := chain.InsertChain(blocks[1:2]); err != nil {
@@ -1932,7 +1932,7 @@ func TestIssue23496(t *testing.T) {
19321932
if _, err := chain.InsertChain(blocks[2:3]); err != nil {
19331933
t.Fatalf("Failed to import canonical chain start: %v", err)
19341934
}
1935-
chain.stateCache.TrieDB().Commit(blocks[2].Root(), true, nil)
1935+
chain.stateCache.TrieDB().Commit(blocks[2].Root(), false)
19361936

19371937
// Insert the remaining blocks
19381938
if _, err := chain.InsertChain(blocks[3:]); err != nil {

core/blockchain_sethead_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2004,7 +2004,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
20042004
t.Fatalf("Failed to import canonical chain start: %v", err)
20052005
}
20062006
if tt.commitBlock > 0 {
2007-
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
2007+
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
20082008
if snapshots {
20092009
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
20102010
t.Fatalf("Failed to flatten snapshots: %v", err)

core/blockchain_snapshot_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
9999
startPoint = point
100100

101101
if basic.commitBlock > 0 && basic.commitBlock == point {
102-
chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil)
102+
chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false)
103103
}
104104
if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
105105
// Flushing the entire snap tree into the disk, the

core/blockchain_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1701,7 +1701,7 @@ func TestTrieForkGC(t *testing.T) {
17011701
chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root())
17021702
chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root())
17031703
}
1704-
if len(chain.stateCache.TrieDB().Nodes()) > 0 {
1704+
if nodes, _ := chain.TrieDB().Size(); nodes > 0 {
17051705
t.Fatalf("stale tries still alive after garbase collection")
17061706
}
17071707
}

core/chain_makers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
289289
if err != nil {
290290
panic(fmt.Sprintf("state write error: %v", err))
291291
}
292-
if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil {
292+
if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
293293
panic(fmt.Sprintf("trie write error: %v", err))
294294
}
295295
return block, b.receipts

core/dao_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
8383
if _, err := bc.InsertChain(blocks); err != nil {
8484
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
8585
}
86-
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
86+
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
8787
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
8888
}
8989
bc.Stop()
@@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
106106
if _, err := bc.InsertChain(blocks); err != nil {
107107
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
108108
}
109-
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
109+
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
110110
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
111111
}
112112
bc.Stop()
@@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
131131
if _, err := bc.InsertChain(blocks); err != nil {
132132
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
133133
}
134-
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
134+
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
135135
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
136136
}
137137
blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})
@@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
149149
if _, err := bc.InsertChain(blocks); err != nil {
150150
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
151151
}
152-
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
152+
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
153153
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
154154
}
155155
blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})

core/genesis.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database) error {
157157
}
158158
// Commit newly generated states into disk if it's not empty.
159159
if root != types.EmptyRootHash {
160-
if err := triedb.Commit(root, true, nil); err != nil {
160+
if err := triedb.Commit(root, true); err != nil {
161161
return err
162162
}
163163
}

core/rawdb/accessors_state.go

Lines changed: 95 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package rawdb
1818

1919
import (
20+
"encoding/binary"
21+
2022
"github.com/ethereum/go-ethereum/common"
2123
"github.com/ethereum/go-ethereum/ethdb"
2224
"github.com/ethereum/go-ethereum/log"
@@ -28,6 +30,17 @@ func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
2830
return data
2931
}
3032

33+
// WritePreimages writes the provided set of preimages to the database.
34+
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
35+
for hash, preimage := range preimages {
36+
if err := db.Put(preimageKey(hash), preimage); err != nil {
37+
log.Crit("Failed to store trie preimage", "err", err)
38+
}
39+
}
40+
preimageCounter.Inc(int64(len(preimages)))
41+
preimageHitCounter.Inc(int64(len(preimages)))
42+
}
43+
3144
// ReadCode retrieves the contract code of the provided code hash.
3245
func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
3346
// Try with the prefixed code scheme first, if not then try with legacy
@@ -48,12 +61,6 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
4861
return data
4962
}
5063

51-
// ReadTrieNode retrieves the trie node of the provided hash.
52-
func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
53-
data, _ := db.Get(hash.Bytes())
54-
return data
55-
}
56-
5764
// HasCode checks if the contract code corresponding to the
5865
// provided code hash is present in the db.
5966
func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool {
@@ -74,47 +81,101 @@ func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
7481
return ok
7582
}
7683

77-
// HasTrieNode checks if the trie node with the provided hash is present in db.
78-
func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
79-
ok, _ := db.Has(hash.Bytes())
80-
return ok
81-
}
82-
83-
// WritePreimages writes the provided set of preimages to the database.
84-
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
85-
for hash, preimage := range preimages {
86-
if err := db.Put(preimageKey(hash), preimage); err != nil {
87-
log.Crit("Failed to store trie preimage", "err", err)
88-
}
89-
}
90-
preimageCounter.Inc(int64(len(preimages)))
91-
preimageHitCounter.Inc(int64(len(preimages)))
92-
}
93-
9484
// WriteCode writes the provided contract code database.
9585
func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
9686
if err := db.Put(codeKey(hash), code); err != nil {
9787
log.Crit("Failed to store contract code", "err", err)
9888
}
9989
}
10090

101-
// WriteTrieNode writes the provided trie node database.
102-
func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
103-
if err := db.Put(hash.Bytes(), node); err != nil {
104-
log.Crit("Failed to store trie node", "err", err)
105-
}
106-
}
107-
10891
// DeleteCode deletes the specified contract code from the database.
10992
func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
11093
if err := db.Delete(codeKey(hash)); err != nil {
11194
log.Crit("Failed to delete contract code", "err", err)
11295
}
11396
}
11497

115-
// DeleteTrieNode deletes the specified trie node from the database.
116-
func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
117-
if err := db.Delete(hash.Bytes()); err != nil {
118-
log.Crit("Failed to delete trie node", "err", err)
98+
// ReadTrieHistory retrieves the trie history with the given id. Calculate
99+
// the real position of trie history in freezer by minus one since the first
100+
// history object is started from one(zero for empty state).
101+
func ReadTrieHistory(db ethdb.AncientReaderOp, id uint64) []byte {
102+
blob, err := db.Ancient(trieHistoryTable, id-1)
103+
if err != nil {
104+
return nil
105+
}
106+
return blob
107+
}
108+
109+
// WriteTrieHistory writes the provided trie history to database. Calculate the
110+
// real position of trie history in freezer by minus one since the first history
111+
// object is started from one(zero is not existent corresponds to empty state).
112+
func WriteTrieHistory(db ethdb.AncientWriter, id uint64, blob []byte) {
113+
db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
114+
op.AppendRaw(trieHistoryTable, id-1, blob)
115+
return nil
116+
})
117+
}
118+
119+
// ReadStateLookup retrieves the state id with the provided state root.
120+
func ReadStateLookup(db ethdb.KeyValueReader, root common.Hash) (uint64, bool) {
121+
data, err := db.Get(stateLookupKey(root))
122+
if err != nil || len(data) == 0 {
123+
return 0, false
124+
}
125+
return binary.BigEndian.Uint64(data), true
126+
}
127+
128+
// WriteStateLookup writes the provided state lookup to database.
129+
func WriteStateLookup(db ethdb.KeyValueWriter, root common.Hash, id uint64) {
130+
var buff [8]byte
131+
binary.BigEndian.PutUint64(buff[:], id)
132+
if err := db.Put(stateLookupKey(root), buff[:]); err != nil {
133+
log.Crit("Failed to store state lookup", "err", err)
134+
}
135+
}
136+
137+
// DeleteStateLookup deletes the specified state lookup from the database.
138+
func DeleteStateLookup(db ethdb.KeyValueWriter, root common.Hash) {
139+
if err := db.Delete(stateLookupKey(root)); err != nil {
140+
log.Crit("Failed to delete state lookup", "err", err)
141+
}
142+
}
143+
144+
// ReadHeadState retrieves the id of the disk state from the database.
145+
func ReadHeadState(db ethdb.KeyValueReader) uint64 {
146+
data, _ := db.Get(headStateKey)
147+
if len(data) != 8 {
148+
return 0
149+
}
150+
return binary.BigEndian.Uint64(data)
151+
}
152+
153+
// WriteHeadState stores the id of the disk state into database.
154+
func WriteHeadState(db ethdb.KeyValueWriter, number uint64) {
155+
if err := db.Put(headStateKey, encodeBlockNumber(number)); err != nil {
156+
log.Crit("Failed to store the head state id", "err", err)
157+
}
158+
}
159+
160+
// ReadTrieJournal retrieves the serialized in-memory trie node diff layers saved at
161+
// the last shutdown. The blob is expected to be max a few 10s of megabytes.
162+
func ReadTrieJournal(db ethdb.KeyValueReader) []byte {
163+
data, _ := db.Get(triesJournalKey)
164+
return data
165+
}
166+
167+
// WriteTrieJournal stores the serialized in-memory trie node diff layers to save at
168+
// shutdown. The blob is expected to be max a few 10s of megabytes.
169+
func WriteTrieJournal(db ethdb.KeyValueWriter, journal []byte) {
170+
if err := db.Put(triesJournalKey, journal); err != nil {
171+
log.Crit("Failed to store tries journal", "err", err)
172+
}
173+
}
174+
175+
// DeleteTrieJournal deletes the serialized in-memory trie node diff layers saved at
176+
// the last shutdown
177+
func DeleteTrieJournal(db ethdb.KeyValueWriter) {
178+
if err := db.Delete(triesJournalKey); err != nil {
179+
log.Crit("Failed to remove tries journal", "err", err)
119180
}
120181
}

0 commit comments

Comments
 (0)