tests: external evm benchmarks (ethereum#24050)

* tests: add ipsilon/evm-benchmarks git submodule

* tests: plug-in evm-benchmarks

.gitmodules

@@ -2,3 +2,7 @@
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
 	shallow = true
+[submodule "evm-benchmarks"]
+	path = tests/evm-benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks
+	shallow = true

tests/init_test.go

@@ -41,6 +41,7 @@ var (
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
 	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
+	benchmarksDir      = filepath.Join(".", "evm-benchmarks", "benchmarks")
 )
 
 func readJSON(reader io.Reader, value interface{}) error {

tests/state_test.go

@@ -20,9 +20,16 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 )
@@ -61,6 +68,7 @@ func TestState(t *testing.T) {
 	for _, dir := range []string{
 		stateTestDir,
 		legacyStateTestDir,
+		benchmarksDir,
 	} {
 		st.walk(t, dir, func(t *testing.T, name string, test *StateTest) {
 			for _, subtest := range test.Subtests() {
@@ -131,3 +139,116 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) {
 	// t.Logf("EVM output: 0x%x", tracer.Output())
 	// t.Logf("EVM error: %v", tracer.Error())
 }
+
+func BenchmarkEVM(b *testing.B) {
+	// Walk the directory.
+	dir := benchmarksDir
+	dirinfo, err := os.Stat(dir)
+	if os.IsNotExist(err) || !dirinfo.IsDir() {
+		fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the evm-benchmarks submodule?\n", dir)
+		b.Skip("missing test files")
+	}
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+		if ext := filepath.Ext(path); ext == ".json" {
+			name := filepath.ToSlash(strings.TrimPrefix(strings.TrimSuffix(path, ext), dir+string(filepath.Separator)))
+			b.Run(name, func(b *testing.B) { runBenchmarkFile(b, path) })
+		}
+		return nil
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+}
+
+func runBenchmarkFile(b *testing.B, path string) {
+	m := make(map[string]StateTest)
+	if err := readJSONFile(path, &m); err != nil {
+		b.Fatal(err)
+		return
+	}
+	if len(m) != 1 {
+		b.Fatal("expected single benchmark in a file")
+		return
+	}
+	for _, t := range m {
+		runBenchmark(b, &t)
+	}
+}
+
+func runBenchmark(b *testing.B, t *StateTest) {
+	for _, subtest := range t.Subtests() {
+		subtest := subtest
+		key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
+
+		b.Run(key, func(b *testing.B) {
+			vmconfig := vm.Config{}
+
+			config, eips, err := GetChainConfig(subtest.Fork)
+			if err != nil {
+				b.Error(err)
+				return
+			}
+			vmconfig.ExtraEips = eips
+			block := t.genesis(config).ToBlock(nil)
+			_, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false)
+
+			var baseFee *big.Int
+			if config.IsLondon(new(big.Int)) {
+				baseFee = t.json.Env.BaseFee
+				if baseFee == nil {
+					// Retesteth uses `0x10` for genesis baseFee. Therefore, it defaults to
+					// parent - 2 : 0xa as the basefee for 'this' context.
+					baseFee = big.NewInt(0x0a)
+				}
+			}
+			post := t.json.Post[subtest.Fork][subtest.Index]
+			msg, err := t.json.Tx.toMessage(post, baseFee)
+			if err != nil {
+				b.Error(err)
+				return
+			}
+
+			// Try to recover tx with current signer
+			if len(post.TxBytes) != 0 {
+				var ttx types.Transaction
+				err := ttx.UnmarshalBinary(post.TxBytes)
+				if err != nil {
+					b.Error(err)
+					return
+				}
+
+				if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil {
+					b.Error(err)
+					return
+				}
+			}
+
+			// Prepare the EVM.
+			txContext := core.NewEVMTxContext(msg)
+			context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase)
+			context.GetHash = vmTestBlockHash
+			context.BaseFee = baseFee
+			evm := vm.NewEVM(context, txContext, statedb, config, vmconfig)
+
+			// Create "contract" for sender to cache code analysis.
+			sender := vm.NewContract(vm.AccountRef(msg.From()), vm.AccountRef(msg.From()),
+				nil, 0)
+
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				// Execute the message.
+				snapshot := statedb.Snapshot()
+				_, _, err = evm.Call(sender, *msg.To(), msg.Data(), msg.Gas(), msg.Value())
+				if err != nil {
+					b.Error(err)
+					return
+				}
+				statedb.RevertToSnapshot(snapshot)
+			}
+
+		})
+	}
+}