Skip to content

Commit

Permalink
Merge upstream PR 28062
Browse files Browse the repository at this point in the history
core, trie: cleanup trie database (#28062)

 Conflicts:
  core/state/statedb.go
Both us and upstream added fields to the end of StateDb. kept both.
tsahee committed Jan 18, 2024
2 parents 3945fec + a7842c9 commit 1ed9f59
Showing 23 changed files with 201 additions and 90 deletions.
6 changes: 6 additions & 0 deletions beacon/engine/gen_epe.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion beacon/engine/types.go
Original file line number Diff line number Diff line change
@@ -86,6 +86,7 @@ type ExecutionPayloadEnvelope struct {
ExecutionPayload *ExecutableData `json:"executionPayload" gencodec:"required"`
BlockValue *big.Int `json:"blockValue" gencodec:"required"`
BlobsBundle *BlobsBundleV1 `json:"blobsBundle"`
Override bool `json:"shouldOverrideBuilder"`
}

type BlobsBundleV1 struct {
@@ -269,7 +270,7 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.
bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(sidecar.Proofs[j][:]))
}
}
return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &bundle}
return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &bundle, Override: false}
}

// ExecutionPayloadBodyV1 is used in the response to GetPayloadBodiesByHashV1 and GetPayloadBodiesByRangeV1
9 changes: 5 additions & 4 deletions cmd/devp2p/nodesetcmd.go
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ import (
"strings"
"time"

"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/p2p/enr"
"github.com/ethereum/go-ethereum/params"
@@ -228,13 +229,13 @@ func ethFilter(args []string) (nodeFilter, error) {
var filter forkid.Filter
switch args[0] {
case "mainnet":
filter = forkid.NewStaticFilter(params.MainnetChainConfig, params.MainnetGenesisHash)
filter = forkid.NewStaticFilter(params.MainnetChainConfig, core.DefaultGenesisBlock().ToBlock())
case "goerli":
filter = forkid.NewStaticFilter(params.GoerliChainConfig, params.GoerliGenesisHash)
filter = forkid.NewStaticFilter(params.GoerliChainConfig, core.DefaultGoerliGenesisBlock().ToBlock())
case "sepolia":
filter = forkid.NewStaticFilter(params.SepoliaChainConfig, params.SepoliaGenesisHash)
filter = forkid.NewStaticFilter(params.SepoliaChainConfig, core.DefaultSepoliaGenesisBlock().ToBlock())
case "holesky":
filter = forkid.NewStaticFilter(params.HoleskyChainConfig, params.HoleskyGenesisHash)
filter = forkid.NewStaticFilter(params.HoleskyChainConfig, core.DefaultHoleskyGenesisBlock().ToBlock())
default:
return nil, fmt.Errorf("unknown network %q", args[0])
}
4 changes: 4 additions & 0 deletions cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
untrusted comment: signature from minisign secret key
RUQkliYstQBOKLK05Sy5f3bVRMBqJT26ABo6Vbp3BNJAVjejoqYCu4GWE/+7qcDfHBqYIniDCbFIUvYEnOHxV6vZ93wO1xJWDQw=
trusted comment: timestamp:1693986492 file:data.json hashed
6Fdw2H+W1ZXK7QXSF77Z5AWC7+AEFAfDmTSxNGylU5HLT1AuSJQmxslj+VjtUBamYCvOuET7plbXza942AlWDw==
18 changes: 14 additions & 4 deletions cmd/geth/version_check_test.go
Original file line number Diff line number Diff line change
@@ -30,17 +30,24 @@ import (
)

func TestVerification(t *testing.T) {
// Signatures generated with `minisign`
t.Run("minisig", func(t *testing.T) {
// For this test, the pubkey is in testdata/minisign.pub
// Signatures generated with `minisign`. Legacy format, not pre-hashed file.
t.Run("minisig-legacy", func(t *testing.T) {
// For this test, the pubkey is in testdata/vcheck/minisign.pub
// (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' )
pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp"
testVerification(t, pub, "./testdata/vcheck/minisig-sigs/")
})
t.Run("minisig-new", func(t *testing.T) {
// For this test, the pubkey is in testdata/vcheck/minisign.pub
// (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' )
// `minisign -S -s ./minisign.sec -m data.json -x ./minisig-sigs-new/data.json.minisig`
pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp"
testVerification(t, pub, "./testdata/vcheck/minisig-sigs-new/")
})
// Signatures generated with `signify-openbsd`
t.Run("signify-openbsd", func(t *testing.T) {
t.Skip("This currently fails, minisign expects 4 lines of data, signify provides only 2")
// For this test, the pubkey is in testdata/signifykey.pub
// For this test, the pubkey is in testdata/vcheck/signifykey.pub
// (the privkey is `signifykey.sec`, if we want to expand this test. Password 'test' )
pub := "RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/"
testVerification(t, pub, "./testdata/vcheck/signify-sigs/")
@@ -58,6 +65,9 @@ func testVerification(t *testing.T, pubkey, sigdir string) {
if err != nil {
t.Fatal(err)
}
if len(files) == 0 {
t.Fatal("Missing tests")
}
for _, f := range files {
sig, err := os.ReadFile(filepath.Join(sigdir, f.Name()))
if err != nil {
12 changes: 12 additions & 0 deletions common/types_test.go
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ import (
"reflect"
"strings"
"testing"
"time"
)

func TestBytesConversion(t *testing.T) {
@@ -583,3 +584,14 @@ func TestAddressEIP55(t *testing.T) {
t.Fatal("Unexpected address after unmarshal")
}
}

func BenchmarkPrettyDuration(b *testing.B) {
var x = PrettyDuration(time.Duration(int64(1203123912312)))
b.Logf("Pre %s", time.Duration(x).String())
var a string
b.ResetTimer()
for i := 0; i < b.N; i++ {
a = x.String()
}
b.Logf("Post %s", a)
}
14 changes: 7 additions & 7 deletions consensus/misc/eip4844/eip4844_test.go
Original file line number Diff line number Diff line change
@@ -45,14 +45,14 @@ func TestCalcExcessBlobGas(t *testing.T) {
// The excess blob gas should decrease by however much the target was
// under-shot, capped at zero.
{params.BlobTxTargetBlobGasPerBlock, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, params.BlobTxTargetBlobGasPerBlock},
{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxBlobGasPerBlob},
{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, 0},
{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxTargetBlobGasPerBlock - params.BlobTxBlobGasPerBlob},
{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, params.BlobTxTargetBlobGasPerBlock - (2 * params.BlobTxBlobGasPerBlob)},
{params.BlobTxBlobGasPerBlob - 1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, 0},
}
for _, tt := range tests {
for i, tt := range tests {
result := CalcExcessBlobGas(tt.excess, tt.blobs*params.BlobTxBlobGasPerBlob)
if result != tt.want {
t.Errorf("excess blob gas mismatch: have %v, want %v", result, tt.want)
t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want)
}
}
}
@@ -63,9 +63,9 @@ func TestCalcBlobFee(t *testing.T) {
blobfee int64
}{
{0, 1},
{1542706, 1},
{1542707, 2},
{10 * 1024 * 1024, 111},
{2314057, 1},
{2314058, 2},
{10 * 1024 * 1024, 23},
}
for i, tt := range tests {
have := CalcBlobFee(tt.excessBlobGas)
22 changes: 9 additions & 13 deletions core/forkid/forkid.go
Original file line number Diff line number Diff line change
@@ -26,7 +26,6 @@ import (
"reflect"
"strings"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
@@ -78,7 +77,7 @@ func NewID(config *params.ChainConfig, genesis *types.Block, head, time uint64)
hash := crc32.ChecksumIEEE(genesis.Hash().Bytes())

// Calculate the current fork checksum and the next fork block
forksByBlock, forksByTime := gatherForks(config)
forksByBlock, forksByTime := gatherForks(config, genesis.Time())
for _, fork := range forksByBlock {
if fork <= head {
// Fork already passed, checksum the previous hash and the fork number
@@ -88,10 +87,6 @@ func NewID(config *params.ChainConfig, genesis *types.Block, head, time uint64)
return ID{Hash: checksumToBytes(hash), Next: fork}
}
for _, fork := range forksByTime {
if fork <= genesis.Time() {
// Fork active in genesis, skip in forkid calculation
continue
}
if fork <= time {
// Fork already passed, checksum the previous hash and fork timestamp
hash = checksumUpdate(hash, fork)
@@ -119,7 +114,7 @@ func NewIDWithChain(chain Blockchain) ID {
func NewFilter(chain Blockchain) Filter {
return newFilter(
chain.Config(),
chain.Genesis().Hash(),
chain.Genesis(),
func() (uint64, uint64) {
head := chain.CurrentHeader()
return head.Number.Uint64(), head.Time
@@ -128,22 +123,22 @@ func NewFilter(chain Blockchain) Filter {
}

// NewStaticFilter creates a filter at block zero.
func NewStaticFilter(config *params.ChainConfig, genesis common.Hash) Filter {
func NewStaticFilter(config *params.ChainConfig, genesis *types.Block) Filter {
head := func() (uint64, uint64) { return 0, 0 }
return newFilter(config, genesis, head)
}

// newFilter is the internal version of NewFilter, taking closures as its arguments
// instead of a chain. The reason is to allow testing it without having to simulate
// an entire blockchain.
func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() (uint64, uint64)) Filter {
func newFilter(config *params.ChainConfig, genesis *types.Block, headfn func() (uint64, uint64)) Filter {
// Calculate the all the valid fork hash and fork next combos
var (
forksByBlock, forksByTime = gatherForks(config)
forksByBlock, forksByTime = gatherForks(config, genesis.Time())
forks = append(append([]uint64{}, forksByBlock...), forksByTime...)
sums = make([][4]byte, len(forks)+1) // 0th is the genesis
)
hash := crc32.ChecksumIEEE(genesis[:])
hash := crc32.ChecksumIEEE(genesis.Hash().Bytes())
sums[0] = checksumToBytes(hash)
for i, fork := range forks {
hash = checksumUpdate(hash, fork)
@@ -244,7 +239,7 @@ func checksumToBytes(hash uint32) [4]byte {

// gatherForks gathers all the known forks and creates two sorted lists out of
// them, one for the block number based forks and the second for the timestamps.
func gatherForks(config *params.ChainConfig) ([]uint64, []uint64) {
func gatherForks(config *params.ChainConfig, genesis uint64) ([]uint64, []uint64) {
// Gather all the fork block numbers via reflection
kind := reflect.TypeOf(params.ChainConfig{})
conf := reflect.ValueOf(config).Elem()
@@ -294,7 +289,8 @@ func gatherForks(config *params.ChainConfig) ([]uint64, []uint64) {
if len(forksByBlock) > 0 && forksByBlock[0] == 0 {
forksByBlock = forksByBlock[1:]
}
if len(forksByTime) > 0 && forksByTime[0] == 0 {
// Skip any forks before genesis.
for len(forksByTime) > 0 && forksByTime[0] <= genesis {
forksByTime = forksByTime[1:]
}
return forksByBlock, forksByTime
2 changes: 1 addition & 1 deletion core/forkid/forkid_test.go
Original file line number Diff line number Diff line change
@@ -357,7 +357,7 @@ func TestValidation(t *testing.T) {
//{params.MainnetChainConfig, 20999999, 1677999999, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, ErrLocalIncompatibleOrStale},
}
for i, tt := range tests {
filter := newFilter(tt.config, params.MainnetGenesisHash, func() (uint64, uint64) { return tt.head, tt.time })
filter := newFilter(tt.config, core.DefaultGenesisBlock().ToBlock(), func() (uint64, uint64) { return tt.head, tt.time })
if err := filter(tt.id); err != tt.err {
t.Errorf("test %d: validation error mismatch: have %v, want %v", i, err, tt.err)
}
8 changes: 4 additions & 4 deletions core/rawdb/accessors_metadata.go
Original file line number Diff line number Diff line change
@@ -111,10 +111,10 @@ const crashesToKeep = 10
func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) {
var uncleanShutdowns crashList
// Read old data
if data, err := db.Get(uncleanShutdownKey); err != nil {
log.Warn("Error reading unclean shutdown markers", "error", err)
} else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
return nil, 0, err
if data, err := db.Get(uncleanShutdownKey); err == nil {
if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
return nil, 0, err
}
}
var discarded = uncleanShutdowns.Discarded
var previous = make([]uint64, len(uncleanShutdowns.Recent))
9 changes: 8 additions & 1 deletion core/state/statedb.go
Original file line number Diff line number Diff line change
@@ -142,6 +142,9 @@ type StateDB struct {
AccountDeleted int
StorageDeleted int

// Testing hooks
onCommit func(states *triestate.Set) // Hook invoked when commit is performed

deterministic bool
}

@@ -1332,13 +1335,17 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
}
if root != origin {
start := time.Now()
if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil {
set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
return common.Hash{}, err
}
s.originalRoot = root
if metrics.EnabledExpensive {
s.TrieDBCommits += time.Since(start)
}
if s.onCommit != nil {
s.onCommit(set)
}
}
// Clear all internal flags at the end of commit operation.
s.accounts = make(map[common.Hash][]byte)
4 changes: 3 additions & 1 deletion core/state/statedb_fuzz_test.go
Original file line number Diff line number Diff line change
@@ -181,7 +181,7 @@ func (test *stateTest) run() bool {
storageList = append(storageList, copy2DSet(states.Storages))
}
disk = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit, PathDB: pathdb.Defaults})
tdb = trie.NewDatabase(disk, &trie.Config{PathDB: pathdb.Defaults})
sdb = NewDatabaseWithNodeDB(disk, tdb)
byzantium = rand.Intn(2) == 0
)
@@ -206,6 +206,8 @@ func (test *stateTest) run() bool {
if err != nil {
panic(err)
}
state.onCommit = onCommit

for i, action := range actions {
if i%test.chunk == 0 && i != 0 {
if byzantium {
18 changes: 10 additions & 8 deletions core/txpool/blobpool/slotter_test.go
Original file line number Diff line number Diff line change
@@ -38,14 +38,16 @@ func TestNewSlotter(t *testing.T) {
2*blobSize + txAvgSize, // 2 blob + some expected tx infos (could be fewer blobs and more tx data)
3*blobSize + txAvgSize, // 3 blob + some expected tx infos (could be fewer blobs and more tx data)
4*blobSize + txAvgSize, // 4 blob + some expected tx infos (could be fewer blobs and more tx data)
5*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
6*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
7*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
8*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
9*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
10*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
11*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
12*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size
5*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
6*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
7*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
8*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
9*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
10*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
11*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
12*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
13*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
14*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size
}
if len(shelves) != len(want) {
t.Errorf("shelves count mismatch: have %d, want %d", len(shelves), len(want))
4 changes: 4 additions & 0 deletions eth/catalyst/simulated_beacon.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
package catalyst

import (
"crypto/rand"
"errors"
"sync"
"time"
@@ -149,10 +150,13 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error {
c.setCurrentState(header.Hash(), *finalizedHash)
}

var random [32]byte
rand.Read(random[:])
fcResponse, err := c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, &engine.PayloadAttributes{
Timestamp: tstamp,
SuggestedFeeRecipient: feeRecipient,
Withdrawals: withdrawals,
Random: random,
})
if err != nil {
return err
Loading

0 comments on commit 1ed9f59

Please sign in to comment.