TEVM status pruning (#2034)

* use trace for TEVM status

* restore testdata

* fix getting contract from plain state

* unwind

* linters

* reproducible build debug

* debug

* debug

* remove debug

accounts/abi/bind/backends/simulated.go

@@ -66,7 +66,7 @@ var (
 type SimulatedBackend struct {
 	m         *stages.MockSentry
 	getHeader func(hash common.Hash, number uint64) *types.Header
-	checkTEVM func(hash common.Hash) (bool, error)
+	checkTEVM func(common.Hash) (bool, error)
 
 	mu              sync.Mutex
 	prependBlock    *types.Block

cmd/integration/commands/refetence_db.go

@@ -35,7 +35,6 @@ var stateBuckets = []string{
 	dbutils.StorageHistoryBucket,
 	dbutils.TxLookupPrefix,
 	dbutils.ContractTEVMCodeBucket,
-	dbutils.ContractTEVMCodeStatusBucket,
 }
 
 var cmdCompareBucket = &cobra.Command{

cmd/integration/commands/snapshot_check.go

@@ -258,7 +258,7 @@ func snapshotCheck(ctx context.Context, db ethdb.RwKV, isNew bool, tmpDir string
 		log.Info("Stage4", "progress", stage4.BlockNumber)
 
 		err = stagedsync.SpawnExecuteBlocksStage(stage4, sync, tx, blockNumber, ch,
-			stagedsync.StageExecuteBlocksCfg(db, false, false, false, 0, batchSize, nil, nil, nil, chainConfig, engine, vmConfig, tmpDir), nil,
+			stagedsync.StageExecuteBlocksCfg(db, false, false, false, 0, batchSize, nil, chainConfig, engine, vmConfig, tmpDir), nil,
 		)
 		if err != nil {
 			return fmt.Errorf("execution err %w", err)

cmd/integration/commands/stages.go

@@ -425,7 +425,7 @@ func stageExec(db ethdb.RwKV, ctx context.Context) error {
 
 	log.Info("Stage4", "progress", execStage.BlockNumber)
 	ch := ctx.Done()
-	cfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, nil, nil, nil, chainConfig, engine, vmConfig, tmpDBPath)
+	cfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, nil, chainConfig, engine, vmConfig, tmpDBPath)
 	if unwind > 0 {
 		u := &stagedsync.UnwindState{Stage: stages.Execution, UnwindPoint: execStage.BlockNumber - unwind}
 		err := stagedsync.UnwindExecutionStage(u, execStage, nil, ch, cfg, nil)

cmd/integration/commands/state_stages.go

@@ -177,7 +177,7 @@ func syncBySmallSteps(db ethdb.RwKV, miningConfig params.MiningConfig, ctx conte
 		stages.TxPool, // TODO: enable TxPool stage
 		stages.Finish)
 
-	execCfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, nil, nil, changeSetHook, chainConfig, engine, vmConfig, tmpDir)
+	execCfg := stagedsync.StageExecuteBlocksCfg(db, sm.Receipts, sm.CallTraces, sm.TEVM, 0, batchSize, changeSetHook, chainConfig, engine, vmConfig, tmpDir)
 
 	execUntilFunc := func(execToBlock uint64) func(stageState *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {
 		return func(s *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {
@@ -498,7 +498,7 @@ func loopExec(db ethdb.RwKV, ctx context.Context, unwind uint64) error {
 
 	from := progress(tx, stages.Execution)
 	to := from + unwind
-	cfg := stagedsync.StageExecuteBlocksCfg(db, true, false, false, 0, batchSize, nil, nil, nil, chainConfig, engine, vmConfig, tmpDBPath)
+	cfg := stagedsync.StageExecuteBlocksCfg(db, true, false, false, 0, batchSize, nil, chainConfig, engine, vmConfig, tmpDBPath)
 
 	// set block limit of execute stage
 	sync.MockExecFunc(stages.Execution, func(stageState *stagedsync.StageState, unwinder stagedsync.Unwinder, tx ethdb.RwTx) error {

cmd/pics/state.go

@@ -68,30 +68,29 @@ import (
 }*/
 
 var bucketLabels = map[string]string{
-	dbutils.BlockReceiptsPrefix:          "Receipts",
-	dbutils.Log:                          "Event Logs",
-	dbutils.AccountsHistoryBucket:        "History Of Accounts",
-	dbutils.StorageHistoryBucket:         "History Of Storage",
-	dbutils.HeadersBucket:                "Headers",
-	dbutils.HeaderCanonicalBucket:        "Canonical headers",
-	dbutils.HeaderTDBucket:               "Headers TD",
-	dbutils.BlockBodyPrefix:              "Block Bodies",
-	dbutils.HeaderNumberBucket:           "Header Numbers",
-	dbutils.TxLookupPrefix:               "Transaction Index",
-	dbutils.CodeBucket:                   "Code Of Contracts",
-	dbutils.SyncStageProgress:            "Sync Progress",
-	dbutils.PlainStateBucket:             "Plain State",
-	dbutils.HashedAccountsBucket:         "Hashed Accounts",
-	dbutils.HashedStorageBucket:          "Hashed Storage",
-	dbutils.TrieOfAccountsBucket:         "Intermediate Hashes Of Accounts",
-	dbutils.TrieOfStorageBucket:          "Intermediate Hashes Of Storage",
-	dbutils.SyncStageUnwind:              "Unwind",
-	dbutils.AccountChangeSetBucket:       "Account Changes",
-	dbutils.StorageChangeSetBucket:       "Storage Changes",
-	dbutils.IncarnationMapBucket:         "Incarnations",
-	dbutils.Senders:                      "Transaction Senders",
-	dbutils.ContractTEVMCodeBucket:       "Contract TEVM code",
-	dbutils.ContractTEVMCodeStatusBucket: "Contract TEVM code status",
+	dbutils.BlockReceiptsPrefix:    "Receipts",
+	dbutils.Log:                    "Event Logs",
+	dbutils.AccountsHistoryBucket:  "History Of Accounts",
+	dbutils.StorageHistoryBucket:   "History Of Storage",
+	dbutils.HeadersBucket:          "Headers",
+	dbutils.HeaderCanonicalBucket:  "Canonical headers",
+	dbutils.HeaderTDBucket:         "Headers TD",
+	dbutils.BlockBodyPrefix:        "Block Bodies",
+	dbutils.HeaderNumberBucket:     "Header Numbers",
+	dbutils.TxLookupPrefix:         "Transaction Index",
+	dbutils.CodeBucket:             "Code Of Contracts",
+	dbutils.SyncStageProgress:      "Sync Progress",
+	dbutils.PlainStateBucket:       "Plain State",
+	dbutils.HashedAccountsBucket:   "Hashed Accounts",
+	dbutils.HashedStorageBucket:    "Hashed Storage",
+	dbutils.TrieOfAccountsBucket:   "Intermediate Hashes Of Accounts",
+	dbutils.TrieOfStorageBucket:    "Intermediate Hashes Of Storage",
+	dbutils.SyncStageUnwind:        "Unwind",
+	dbutils.AccountChangeSetBucket: "Account Changes",
+	dbutils.StorageChangeSetBucket: "Storage Changes",
+	dbutils.IncarnationMapBucket:   "Incarnations",
+	dbutils.Senders:                "Transaction Senders",
+	dbutils.ContractTEVMCodeBucket: "Contract TEVM code",
 }
 
 /*dbutils.PlainContractCodeBucket,

cmd/rpcdaemon/commands/trace_adhoc.go

@@ -164,7 +164,7 @@ type OeTracer struct {
 	precompile bool // Whether the last CaptureStart was called with `precompile = true`
 }
 
-func (ot *OeTracer) CaptureStart(depth int, from common.Address, to common.Address, precompile bool, create bool, calltype vm.CallType, input []byte, gas uint64, value *big.Int) error {
+func (ot *OeTracer) CaptureStart(depth int, from common.Address, to common.Address, precompile bool, create bool, calltype vm.CallType, input []byte, gas uint64, value *big.Int, codeHash common.Hash) error {
 	if precompile {
 		ot.precompile = true
 		return nil

cmd/sentry/download/sentry.go

@@ -924,7 +924,7 @@ func (ss *SentryServerImpl) addStream(ids []proto_sentry.MessageId, server proto
 }
 
 func (ss *SentryServerImpl) Messages(req *proto_sentry.MessagesRequest, server proto_sentry.Sentry_MessagesServer) error {
-	log.Info(fmt.Sprintf("[Messages] new subscriber to: %s\n", req.Ids))
+	log.Info(fmt.Sprintf("[Messages] new subscriber to: %s", req.Ids))
 	clean := ss.addStream(req.Ids, server)
 	defer clean()
 	select {

cmd/state/commands/opcode_tracer.go

@@ -152,7 +152,7 @@ type blockTxs struct {
 	Txs      slicePtrTx
 }
 
-func (ot *opcodeTracer) CaptureStart(depth int, from common.Address, to common.Address, precompile bool, create bool, calltype vm.CallType, input []byte, gas uint64, value *big.Int) error {
+func (ot *opcodeTracer) CaptureStart(depth int, from common.Address, to common.Address, precompile bool, create bool, calltype vm.CallType, input []byte, gas uint64, value *big.Int, codeHash common.Hash) error {
 	//fmt.Fprint(ot.summary, ot.lastLine)
 
 	// When a CaptureStart is called, a Tx is starting. Create its entry in our list and initialize it with the partial data available
@@ -662,7 +662,7 @@ func check(e error) {
 }
 
 func runBlock(ibs *state.IntraBlockState, txnWriter state.StateWriter, blockWriter state.StateWriter,
-	chainConfig *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, checkTEVM func(hash common.Hash) (bool, error), block *types.Block, vmConfig vm.Config) (types.Receipts, error) {
+	chainConfig *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, checkTEVM func(common.Hash) (bool, error), block *types.Block, vmConfig vm.Config) (types.Receipts, error) {
 	header := block.Header()
 	vmConfig.TraceJumpDest = true
 	engine := ethash.NewFullFaker()

common/dbutils/bucket.go

@@ -140,14 +140,9 @@ var (
 	//value - incarnation of account when it was last deleted
 	IncarnationMapBucket = "incarnationMap"
 
-	//TEVMCodeStatusBucket -
-	//key - encoded timestamp(block number)
-	//value - contract codes hashes: [code_hash1]+[code_hash2]
-	ContractTEVMCodeStatusBucket = "TEVMCodeStatus"
-
 	//TEVMCodeBucket -
 	//key - contract code hash
-	//value - contract EVTM code
+	//value - contract TEVM code
 	ContractTEVMCodeBucket = "TEVMCode"
 )
 
@@ -230,7 +225,7 @@ const (
 
 	// CallTraceSet is the name of the table that contain the mapping of block number to the set (sorted) of all accounts
 	// touched by call traces. It is DupSort-ed table
-	// 8-byte BE block nunber -> account address -> two bits (one for "from", another for "to")
+	// 8-byte BE block number -> account address -> two bits (one for "from", another for "to")
 	CallTraceSet = "call_trace_set"
 	// Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex
 	// Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses
@@ -372,7 +367,6 @@ var Buckets = []string{
 	DatabaseInfoBucket,
 	IncarnationMapBucket,
 	ContractTEVMCodeBucket,
-	ContractTEVMCodeStatusBucket,
 	CliqueSeparateBucket,
 	CliqueLastSnapshotBucket,
 	CliqueSnapshotBucket,
@@ -503,9 +497,6 @@ var BucketsConfigs = BucketsCfg{
 	CallTraceSet: {
 		Flags: DupSort,
 	},
-	ContractTEVMCodeStatusBucket: {
-		Flags: DupSort,
-	},
 }
 
 func sortBuckets() {

core/blockchain.go

@@ -94,7 +94,7 @@ func ExecuteBlockEphemerally(
 	block *types.Block,
 	stateReader state.StateReader,
 	stateWriter state.WriterWithChangeSets,
-	checkTEVM func(hash common.Hash) (bool, error),
+	checkTEVM func(codeHash common.Hash) (bool, error),
 ) (types.Receipts, error) {
 	defer blockExecutionTimer.UpdateSince(time.Now())
 	block.Uncles()

core/chain_makers.go

@@ -110,7 +110,7 @@ func (b *BlockGen) AddTxWithChain(getHeader func(hash common.Hash, number uint64
 		b.SetCoinbase(common.Address{})
 	}
 	b.ibs.Prepare(tx.Hash(), common.Hash{}, len(b.txs))
-	checkTEVM := func(common.Hash) (bool, error) { return false, nil }
+	checkTEVM := func(_ common.Hash) (bool, error) { return false, nil }
 	receipt, err := ApplyTransaction(b.config, getHeader, engine, &b.header.Coinbase, b.gasPool, b.ibs, state.NewNoopWriter(), b.header, tx, &b.header.GasUsed, vm.Config{}, checkTEVM)
 	if err != nil {
 		panic(err)

core/evm.go

@@ -28,7 +28,7 @@ import (
 )
 
 // NewEVMBlockContext creates a new context for use in the EVM.
-func NewEVMBlockContext(header *types.Header, getHeader func(hash common.Hash, number uint64) *types.Header, engine consensus.Engine, author *common.Address, checkTEVM func(codeHash common.Hash) (bool, error)) vm.BlockContext {
+func NewEVMBlockContext(header *types.Header, getHeader func(hash common.Hash, number uint64) *types.Header, engine consensus.Engine, author *common.Address, checkTEVM func(contractHash common.Hash) (bool, error)) vm.BlockContext {
 	// If we don't have an explicit author (i.e. not mining), extract from the header
 	var beneficiary common.Address
 	if author == nil {

core/state_processor.go

@@ -147,7 +147,7 @@ func applyTransaction(msg types.Message, config *params.ChainConfig, getHeader f
 // and uses the input parameters for its environment. It returns the receipt
 // for the transaction, gas used and an error if the transaction failed,
 // indicating the block was invalid.
-func ApplyTransaction(config *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, engine consensus.Engine, author *common.Address, gp *GasPool, ibs *state.IntraBlockState, stateWriter state.StateWriter, header *types.Header, tx types.Transaction, usedGas *uint64, cfg vm.Config, checkTEVM func(hash common.Hash) (bool, error)) (*types.Receipt, error) {
+func ApplyTransaction(config *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, engine consensus.Engine, author *common.Address, gp *GasPool, ibs *state.IntraBlockState, stateWriter state.StateWriter, header *types.Header, tx types.Transaction, usedGas *uint64, cfg vm.Config, checkTEVM func(contractHash common.Hash) (bool, error)) (*types.Receipt, error) {
 	msg, err := tx.AsMessage(*types.MakeSigner(config, header.Number.Uint64()), header.BaseFee)
 	if err != nil {
 		return nil, err

core/types/accounts/account.go

@@ -543,7 +543,11 @@ func (a *Account) DecodeRLP(s *rlp.Stream) error {
 }
 
 func (a *Account) IsEmptyCodeHash() bool {
-	return a.CodeHash == emptyCodeHash || a.CodeHash == (common.Hash{})
+	return IsEmptyCodeHash(a.CodeHash)
+}
+
+func IsEmptyCodeHash(codeHash common.Hash) bool {
+	return codeHash == emptyCodeHash || codeHash == (common.Hash{})
 }
 
 func (a *Account) IsEmptyRoot() bool {

core/vm/evm.go

@@ -228,7 +228,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 	p, isPrecompile := evm.precompile(addr)
 	// Capture the tracer start/end events in debug mode
 	if evm.vmConfig.Debug {
-		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, CALLT, input, gas, value.ToBig())
+		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, CALLT, input, gas, value.ToBig(), common.Hash{})
 		defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters
 			evm.vmConfig.Tracer.CaptureEnd(evm.depth, ret, startGas-gas, time.Since(startTime), err) //nolint:errcheck
 		}(gas, time.Now())
@@ -311,7 +311,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 	p, isPrecompile := evm.precompile(addr)
 	// Capture the tracer start/end events in debug mode
 	if evm.vmConfig.Debug {
-		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, CALLCODET, input, gas, value.ToBig())
+		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, CALLCODET, input, gas, value.ToBig(), common.Hash{})
 		defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters
 			evm.vmConfig.Tracer.CaptureEnd(evm.depth, ret, startGas-gas, time.Since(startTime), err) //nolint:errcheck
 		}(gas, time.Now())
@@ -364,7 +364,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by
 	p, isPrecompile := evm.precompile(addr)
 	// Capture the tracer start/end events in debug mode
 	if evm.vmConfig.Debug {
-		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, DELEGATECALLT, input, gas, big.NewInt(-1))
+		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false /* create */, DELEGATECALLT, input, gas, big.NewInt(-1), common.Hash{})
 		defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters
 			evm.vmConfig.Tracer.CaptureEnd(evm.depth, ret, startGas-gas, time.Since(startTime), err) //nolint:errcheck
 		}(gas, time.Now())
@@ -412,7 +412,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	p, isPrecompile := evm.precompile(addr)
 	// Capture the tracer start/end events in debug mode
 	if evm.vmConfig.Debug {
-		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false, STATICCALLT, input, gas, big.NewInt(-2))
+		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), addr, isPrecompile, false, STATICCALLT, input, gas, big.NewInt(-2), common.Hash{})
 		defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters
 			evm.vmConfig.Tracer.CaptureEnd(evm.depth, ret, startGas-gas, time.Since(startTime), err) //nolint:errcheck
 		}(gas, time.Now())
@@ -486,8 +486,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
 	if !evm.Context.CanTransfer(evm.IntraBlockState, caller.Address(), value) {
 		return nil, common.Address{}, gas, ErrInsufficientBalance
 	}
-	if evm.vmConfig.Debug {
-		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), address, false /* precompile */, true /* create */, calltype, codeAndHash.code, gas, value.ToBig())
+	if evm.vmConfig.Debug || evm.vmConfig.EnableTEMV {
+		_ = evm.vmConfig.Tracer.CaptureStart(evm.depth, caller.Address(), address, false /* precompile */, true /* create */, calltype, codeAndHash.code, gas, value.ToBig(), codeAndHash.Hash())
 		defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters
 			evm.vmConfig.Tracer.CaptureEnd(evm.depth, ret, startGas-gas, time.Since(startTime), err) //nolint:errcheck
 		}(gas, time.Now())

core/vm/instructions_test.go

@@ -96,7 +96,7 @@ func init() {
 func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) {
 	var (
 		env = NewEVM(BlockContext{
-			CheckTEVM: func(h common.Hash) (bool, error) { return false, nil },
+			CheckTEVM: func(common.Hash) (bool, error) { return false, nil },
 		}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = stack.New()
 		pc             = uint64(0)
@@ -197,7 +197,7 @@ func TestSAR(t *testing.T) {
 func TestAddMod(t *testing.T) {
 	var (
 		env = NewEVM(BlockContext{
-			CheckTEVM: func(h common.Hash) (bool, error) { return false, nil },
+			CheckTEVM: func(common.Hash) (bool, error) { return false, nil },
 		}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = stack.New()
 		evmInterpreter = NewEVMInterpreter(env, env.vmConfig)
@@ -286,7 +286,7 @@ func TestJsonTestcases(t *testing.T) {
 func opBenchmark(bench *testing.B, op executionFunc, args ...string) {
 	var (
 		env = NewEVM(BlockContext{
-			CheckTEVM: func(h common.Hash) (bool, error) { return false, nil },
+			CheckTEVM: func(common.Hash) (bool, error) { return false, nil },
 		}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = stack.New()
 		evmInterpreter = NewEVMInterpreter(env, env.vmConfig)
@@ -522,7 +522,7 @@ func BenchmarkOpIsZero(b *testing.B) {
 func TestOpMstore(t *testing.T) {
 	var (
 		env = NewEVM(BlockContext{
-			CheckTEVM: func(h common.Hash) (bool, error) { return false, nil },
+			CheckTEVM: func(common.Hash) (bool, error) { return false, nil },
 		}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = stack.New()
 		mem            = NewMemory()
@@ -548,7 +548,7 @@ func TestOpMstore(t *testing.T) {
 func BenchmarkOpMstore(bench *testing.B) {
 	var (
 		env = NewEVM(BlockContext{
-			CheckTEVM: func(h common.Hash) (bool, error) { return false, nil },
+			CheckTEVM: func(common.Hash) (bool, error) { return false, nil },
 		}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = stack.New()
 		mem            = NewMemory()

core/vm/interpreter.go

@@ -36,6 +36,7 @@ type Config struct {
 	TraceJumpDest           bool   // Print transaction hashes where jumpdest analysis was useful
 	NoReceipts              bool   // Do not calculate receipts
 	ReadOnly                bool   // Do no perform any block finalisation
+	EnableTEMV              bool   // true if execution with TEVM enable flag
 
 	ExtraEips []int // Additional EIPS that are to be enabled
 }