Skip to content

Commit

Permalink
all: implement EIP-compliant verkle trees
Browse files Browse the repository at this point in the history
verkle: Implement Trie, NodeIterator and Database ifs

Fix crash in TestDump

Fix TestDump

Fix TrieCopy

remove unnecessary traces

fix: Error() returned errIteratorEnd in verkle node iterator

rewrite the iterator and change the signature of OpenStorageTrie

add the adapter to reuse the account trie for storage

don't try to deserialize a storage leaf into an account

Fix statedb unit tests (ethereum#14)

* debug code

* Fix more unit tests

* remove traces

* Go back to the full range

One tree to rule them all

remove updateRoot, there is no root to update

store code inside the account leaf

fix build

save current state for Sina

Update go-verkle to latest

Charge WITNESS_*_COST gas on storage loads

Add witness costs for SSTORE as well

Charge witness gas in the case of code execution

corresponding code deletion

add a --verkle flag to separate verkle experiments from regular geth operations

use the snapshot to get data

stateless execution from block witness

AccessWitness functions

Add block generation test + genesis snapshot generation

test stateless block execution (ethereum#18)

* test stateless block execution

* Force tree resolution before generating the proof

increased coverage in stateless test execution (ethereum#19)

* test stateless block execution

* Force tree resolution before generating the proof

* increase coverage in stateless test execution

ensure geth compiles

fix issues in tests with verkle trees deactivated

Ensure stateless data is available when executing statelessly (ethereum#20)

* Ensure stateless data is available when executing statelessly

* Actual execution of a statless block

* bugfixes in stateless block execution

* code cleanup

 - Reduce PR footprint by reverting NewEVM to its original signature
 - Move the access witness to the block context
 - prepare for a change in AW semantics
   Need to store the initial values.
 - Use the touch helper function, DRY

* revert the signature of MustCommit to its original form (ethereum#21)

fix leaf proofs in stateless execution (ethereum#22)

* Fixes in witness pre-state

* Add the recipient's nonce to the witness

* reduce PR footprint and investigate issue in root state calculation

* quick build fix

cleanup: Remove extra parameter in ToBlock

revert ToBlock to its older signature

fix import cycle in vm tests

fix linter issue

fix appveyor build

fix nil pointers in tests

Add indices, yis and Cis to the block's Verkle proof

upgrade geth dependency to drop geth's common dep

fix cmd/devp2p tests

fix rebase issues

quell an appveyor warning

fix address touching in SLOAD and SSTORE

fix access witness for code size

touch target account data before calling

make sure the proper locations get touched in (ext)codecopy

touch all code pages in execution

add pushdata to witness

remove useless code in genesis snapshot generation

testnet: fix some of the rebase/drift issues

Fix verkle proof generation in block

fix an issue occuring when chunking past the code size

fix: ensure the code copy doesn't extend past the code size
  • Loading branch information
gballet committed Nov 12, 2021
1 parent e9294a7 commit 6a1f344
Show file tree
Hide file tree
Showing 43 changed files with 1,731 additions and 65 deletions.
2 changes: 1 addition & 1 deletion accounts/abi/bind/bind_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1983,7 +1983,7 @@ func TestGolangBindings(t *testing.T) {
t.Fatalf("failed to tidy Go module file: %v\n%s", err, out)
}
// Test the entire package and report any failures
cmd := exec.Command(gocmd, "test", "-v", "-count", "1")
cmd := exec.Command(gocmd, "test", "-tags=bignum_kilic", "-v", "-count", "1")
cmd.Dir = pkg
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("failed to run binding test: %v\n%s", err, out)
Expand Down
5 changes: 5 additions & 0 deletions cmd/geth/chaincmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ var (
ArgsUsage: "<genesisPath>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.VerkleFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
Expand Down Expand Up @@ -205,6 +206,10 @@ func initGenesis(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

if ctx.GlobalBool(utils.VerkleFlag.Name) {
genesis.Config.UseVerkle = true
}

for _, name := range []string{"chaindata", "lightchaindata"} {
chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions cmd/geth/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ var (
utils.MinerNotifyFullFlag,
configFileFlag,
utils.CatalystFlag,
utils.VerkleFlag,
}

rpcFlags = []cli.Flag{
Expand Down
4 changes: 2 additions & 2 deletions cmd/geth/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func verifyState(ctx *cli.Context) error {
log.Error("Failed to load head block")
return errors.New("no head block")
}
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false)
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false, false)
if err != nil {
log.Error("Failed to open snapshot tree", "err", err)
return err
Expand Down Expand Up @@ -472,7 +472,7 @@ func dumpState(ctx *cli.Context) error {
if err != nil {
return err
}
snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false)
snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false, false)
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions cmd/geth/usage.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
utils.BloomFilterSizeFlag,
cli.HelpFlag,
utils.CatalystFlag,
utils.VerkleFlag,
},
},
}
Expand Down
16 changes: 16 additions & 0 deletions cmd/utils/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,11 @@ var (
Name: "catalyst",
Usage: "Catalyst mode (eth2 integration testing)",
}

VerkleFlag = cli.BoolFlag{
Name: "verkle",
Usage: "Enable geth with verkle trees (EXPERIMENTAL)",
}
)

// MakeDataDir retrieves the currently requested data directory, terminating
Expand Down Expand Up @@ -1465,7 +1470,11 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Avoid conflicting network flags
<<<<<<< HEAD
CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag)
=======
CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, VerkleFlag)
>>>>>>> 00ed1c501 (all: implement EIP-compliant verkle trees)
CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 {
Expand Down Expand Up @@ -1603,6 +1612,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
}
cfg.Genesis = core.DefaultGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
case ctx.GlobalBool(VerkleFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 86 // 'V'
}
cfg.Genesis = core.DefaultVerkleGenesisBlock()
cfg.Genesis.Config.UseVerkle = true
SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
case ctx.GlobalBool(RopstenFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 3
Expand Down
20 changes: 18 additions & 2 deletions core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
Cache: cacheConfig.TrieCleanLimit,
Journal: cacheConfig.TrieCleanJournal,
Preimages: cacheConfig.Preimages,
UseVerkle: chainConfig.UseVerkle,
}),
quit: make(chan struct{}),
chainmu: syncx.NewClosableMutex(),
Expand Down Expand Up @@ -377,7 +378,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
recover = true
}
bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover)
bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.Config().UseVerkle)
}

// Start future block processor.
Expand Down Expand Up @@ -1607,7 +1608,22 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er

// Process block using the parent state as reference point
substart := time.Now()
receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig)
var (
usedGas uint64
receipts types.Receipts
logs []*types.Log
)
if len(block.Header().VerkleProof) == 0 {
receipts, logs, _, usedGas, err = bc.processor.Process(block, statedb, bc.vmConfig)
} else {
var leaves map[common.Hash]common.Hash
_, _, _, leaves, err = trie.DeserializeAndVerifyVerkleProof(block.Header().VerkleProof)
if err != nil {
return it.index, err
}
statedb.SetStateless(leaves)
receipts, logs, usedGas, err = bc.processor.ProcessStateless(block, statedb, bc.vmConfig, leaves)
}
if err != nil {
bc.reportBlock(block, receipts, err)
atomic.StoreUint32(&followupInterrupt, 1)
Expand Down
2 changes: 1 addition & 1 deletion core/blockchain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
if err != nil {
return err
}
receipts, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{})
receipts, _, _, usedGas, err := blockchain.processor.Process(block, statedb, vm.Config{})
if err != nil {
blockchain.reportBlock(block, receipts, err)
return err
Expand Down
95 changes: 94 additions & 1 deletion core/chain_makers.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)

// BlockGen creates blocks for testing.
Expand All @@ -43,6 +44,7 @@ type BlockGen struct {
txs []*types.Transaction
receipts []*types.Receipt
uncles []*types.Header
witness *types.AccessWitness

config *params.ChainConfig
engine consensus.Engine
Expand Down Expand Up @@ -103,10 +105,17 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) {
b.SetCoinbase(common.Address{})
}
b.statedb.Prepare(tx.Hash(), len(b.txs))
receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
receipt, accesses, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{})
if err != nil {
panic(err)
}
if accesses != nil {
if b.witness != nil {
b.witness.Merge(accesses)
} else {
b.witness = accesses
}
}
b.txs = append(b.txs, tx)
b.receipts = append(b.receipts, receipt)
}
Expand Down Expand Up @@ -250,6 +259,90 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
return blocks, receipts
}

func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) {
if config == nil {
config = params.TestChainConfig
}
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
chainreader := &fakeChainReader{config: config}
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine, witness: types.NewAccessWitness()}
b.header = makeHeader(chainreader, parent, statedb, b.engine)

// Mutate the state and block according to any hard-fork specs
if daoBlock := config.DAOForkBlock; daoBlock != nil {
limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange)
if b.header.Number.Cmp(daoBlock) >= 0 && b.header.Number.Cmp(limit) < 0 {
if config.DAOForkSupport {
b.header.Extra = common.CopyBytes(params.DAOForkBlockExtra)
}
}
}
if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 {
misc.ApplyDAOHardFork(statedb)
}
// Execute any user modifications to the block
if gen != nil {
gen(i, b)
}
if b.engine != nil {
// Finalize and seal the block
block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
if err != nil {
panic(err)
}

// Write state changes to db
root, err := statedb.Commit(config.IsEIP158(b.header.Number))
if err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}
if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil {
panic(fmt.Sprintf("trie write error: %v", err))
}

// Generate an associated verkle proof
if tr := statedb.GetTrie(); tr.IsVerkle() {
vtr := tr.(*trie.VerkleTrie)
// Generate the proof if we are using a verkle tree
// WORKAROUND: make sure all keys are resolved
// before building the proof. Ultimately, node
// resolution can be done with a prefetcher or
// from GetCommitmentsAlongPath.
keys := b.witness.Keys()
for _, key := range keys {
out, err := vtr.TryGet(key)
if err != nil {
panic(err)
}
if len(out) == 0 {
panic(fmt.Sprintf("%x should be present in the tree", key))
}
}
vtr.Hash()
_, err := vtr.ProveAndSerialize(keys, b.witness.KeyVals())
//block.SetVerkleProof(p)
if err != nil {
panic(err)
}
}
return block, b.receipts
}
return nil, nil
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabaseWithConfig(db, &trie.Config{UseVerkle: true}), nil)
if err != nil {
panic(err)
}
block, receipt := genblock(i, parent, statedb)
blocks[i] = block
receipts[i] = receipt
parent = block
}
return blocks, receipts
}

func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header {
var time uint64
if parent.Time() == 0 {
Expand Down
1 change: 1 addition & 0 deletions core/evm.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func NewEVMTxContext(msg Message) vm.TxContext {
return vm.TxContext{
Origin: msg.From(),
GasPrice: new(big.Int).Set(msg.GasPrice()),
Accesses: types.NewAccessWitness(),
}
}

Expand Down
29 changes: 27 additions & 2 deletions core/genesis.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
// We have the genesis block in database(perhaps in ancient database)
// but the corresponding state is missing.
header := rawdb.ReadHeader(db, stored, 0)
if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, nil), nil); err != nil {
var cfg *trie.Config = nil
if genesis.Config.UseVerkle {
cfg = &trie.Config{UseVerkle: true}
}
if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, cfg), nil); err != nil {
if genesis == nil {
genesis = DefaultGenesisBlock()
}
Expand Down Expand Up @@ -261,7 +265,11 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
if db == nil {
db = rawdb.NewMemoryDatabase()
}
statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil)
var trieCfg *trie.Config
if g.Config != nil {
trieCfg = &trie.Config{UseVerkle: g.Config.UseVerkle}
}
statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), nil)
if err != nil {
panic(err)
}
Expand Down Expand Up @@ -303,6 +311,9 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
}
statedb.Commit(false)
statedb.Database().TrieDB().Commit(root, true, nil)
if err := statedb.Cap(root); err != nil {
panic(err)
}

return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil))
}
Expand Down Expand Up @@ -354,6 +365,20 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big
return g.MustCommit(db)
}

func DefaultVerkleGenesisBlock() *Genesis {
return &Genesis{
Config: params.VerkleChainConfig,
Nonce: 86,
GasLimit: 0x2fefd8,
Difficulty: big.NewInt(1),
Alloc: map[common.Address]GenesisAccount{
common.BytesToAddress([]byte{97, 118, 97, 209, 72, 165, 43, 239, 81, 162, 104, 199, 40, 179, 162, 27, 88, 249, 67, 6}): {
Balance: big.NewInt(0).Lsh(big.NewInt(1), 27),
},
},
}
}

// DefaultGenesisBlock returns the Ethereum main net genesis block.
func DefaultGenesisBlock() *Genesis {
return &Genesis{
Expand Down
Loading

0 comments on commit 6a1f344

Please sign in to comment.