tests: update execution spec tests + split statetest exec (#28993)

covalenthq · Feb 15, 2024 · 886f0e7 · 886f0e7
1 parent 9e3e466
commit 886f0e7
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 75 deletions.
diff --git a/build/checksums.txt b/build/checksums.txt
@@ -1,9 +1,9 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-# version:spec-tests 1.0.6
+# version:spec-tests 2.1.0
 # https://github.com/ethereum/execution-spec-tests/releases
-# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/
-485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1  fixtures_develop.tar.gz
+# https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.0/
+ca89c76851b0900bfcc3cbb9a26cbece1f3d7c64a3bed38723e914713290df6c  fixtures_develop.tar.gz
 
 # version:golang 1.21.6
 # https://go.dev/dl/

diff --git a/tests/block_test.go b/tests/block_test.go
@@ -61,14 +61,14 @@ func TestBlockchain(t *testing.T) {
 	// which run natively, so there's no reason to run them here.
 }
 
-// TestExecutionSpec runs the test fixtures from execution-spec-tests.
-func TestExecutionSpec(t *testing.T) {
-	if !common.FileExist(executionSpecDir) {
-		t.Skipf("directory %s does not exist", executionSpecDir)
+// TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests.
+func TestExecutionSpecBlocktests(t *testing.T) {
+	if !common.FileExist(executionSpecBlockchainTestDir) {
+		t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir)
 	}
 	bt := new(testMatcher)
 
-	bt.walk(t, executionSpecDir, func(t *testing.T, name string, test *BlockTest) {
+	bt.walk(t, executionSpecBlockchainTestDir, func(t *testing.T, name string, test *BlockTest) {
 		execBlockTest(t, bt, test)
 	})
 }

diff --git a/tests/init_test.go b/tests/init_test.go
@@ -34,15 +34,16 @@ import (
 )
 
 var (
-	baseDir            = filepath.Join(".", "testdata")
-	blockTestDir       = filepath.Join(baseDir, "BlockchainTests")
-	stateTestDir       = filepath.Join(baseDir, "GeneralStateTests")
-	legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests")
-	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
-	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
-	difficultyTestDir  = filepath.Join(baseDir, "BasicTests")
-	executionSpecDir   = filepath.Join(".", "spec-tests", "fixtures")
-	benchmarksDir      = filepath.Join(".", "evm-benchmarks", "benchmarks")
+	baseDir                        = filepath.Join(".", "testdata")
+	blockTestDir                   = filepath.Join(baseDir, "BlockchainTests")
+	stateTestDir                   = filepath.Join(baseDir, "GeneralStateTests")
+	legacyStateTestDir             = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests")
+	transactionTestDir             = filepath.Join(baseDir, "TransactionTests")
+	rlpTestDir                     = filepath.Join(baseDir, "RLPTests")
+	difficultyTestDir              = filepath.Join(baseDir, "BasicTests")
+	executionSpecBlockchainTestDir = filepath.Join(".", "spec-tests", "fixtures", "blockchain_tests")
+	executionSpecStateTestDir      = filepath.Join(".", "spec-tests", "fixtures", "state_tests")
+	benchmarksDir                  = filepath.Join(".", "evm-benchmarks", "benchmarks")
 )
 
 func readJSON(reader io.Reader, value interface{}) error {

diff --git a/tests/state_test.go b/tests/state_test.go
@@ -30,6 +30,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -38,10 +39,7 @@ import (
 	"github.com/holiman/uint256"
 )
 
-func TestState(t *testing.T) {
-	t.Parallel()
-
-	st := new(testMatcher)
+func initMatcher(st *testMatcher) {
 	// Long tests:
 	st.slow(`^stAttackTest/ContractCreationSpam`)
 	st.slow(`^stBadOpcode/badOpcodes`)
@@ -60,72 +58,102 @@ func TestState(t *testing.T) {
 	// Broken tests:
 	// EOF is not part of cancun
 	st.skipLoad(`^stEOF/`)
+}
 
-	// For Istanbul, older tests were moved into LegacyTests
+func TestState(t *testing.T) {
+	t.Parallel()
+
+	st := new(testMatcher)
+	initMatcher(st)
 	for _, dir := range []string{
 		filepath.Join(baseDir, "EIPTests", "StateTests"),
 		stateTestDir,
-		legacyStateTestDir,
 		benchmarksDir,
 	} {
 		st.walk(t, dir, func(t *testing.T, name string, test *StateTest) {
-			if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 {
-				t.Skip("test (randomly) skipped on 32-bit windows")
-				return
-			}
-			for _, subtest := range test.Subtests() {
-				subtest := subtest
-				key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
+			execStateTest(t, st, test)
+		})
+	}
+}
 
-				t.Run(key+"/hash/trie", func(t *testing.T) {
-					withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
-						var result error
-						test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) {
-							result = st.checkFailure(t, err)
-						})
-						return result
-					})
+// TestLegacyState tests some older tests, which were moved to the folder
+// 'LegacyTests' for the Istanbul fork.
+func TestLegacyState(t *testing.T) {
+	st := new(testMatcher)
+	initMatcher(st)
+	st.walk(t, legacyStateTestDir, func(t *testing.T, name string, test *StateTest) {
+		execStateTest(t, st, test)
+	})
+}
+
+// TestExecutionSpecState runs the test fixtures from execution-spec-tests.
+func TestExecutionSpecState(t *testing.T) {
+	if !common.FileExist(executionSpecStateTestDir) {
+		t.Skipf("directory %s does not exist", executionSpecStateTestDir)
+	}
+	st := new(testMatcher)
+
+	st.walk(t, executionSpecStateTestDir, func(t *testing.T, name string, test *StateTest) {
+		execStateTest(t, st, test)
+	})
+}
+
+func execStateTest(t *testing.T, st *testMatcher, test *StateTest) {
+	if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 {
+		t.Skip("test (randomly) skipped on 32-bit windows")
+		return
+	}
+	for _, subtest := range test.Subtests() {
+		subtest := subtest
+		key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
+
+		t.Run(key+"/hash/trie", func(t *testing.T) {
+			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
+				var result error
+				test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) {
+					result = st.checkFailure(t, err)
 				})
-				t.Run(key+"/hash/snap", func(t *testing.T) {
-					withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
-						var result error
-						test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) {
-							if state.Snapshots != nil && state.StateDB != nil {
-								if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil {
-									result = err
-									return
-								}
-							}
-							result = st.checkFailure(t, err)
-						})
-						return result
-					})
+				return result
+			})
+		})
+		t.Run(key+"/hash/snap", func(t *testing.T) {
+			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
+				var result error
+				test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) {
+					if state.Snapshots != nil && state.StateDB != nil {
+						if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil {
+							result = err
+							return
+						}
+					}
+					result = st.checkFailure(t, err)
 				})
-				t.Run(key+"/path/trie", func(t *testing.T) {
-					withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
-						var result error
-						test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) {
-							result = st.checkFailure(t, err)
-						})
-						return result
-					})
+				return result
+			})
+		})
+		t.Run(key+"/path/trie", func(t *testing.T) {
+			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
+				var result error
+				test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) {
+					result = st.checkFailure(t, err)
 				})
-				t.Run(key+"/path/snap", func(t *testing.T) {
-					withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
-						var result error
-						test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) {
-							if state.Snapshots != nil && state.StateDB != nil {
-								if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil {
-									result = err
-									return
-								}
-							}
-							result = st.checkFailure(t, err)
-						})
-						return result
-					})
+				return result
+			})
+		})
+		t.Run(key+"/path/snap", func(t *testing.T) {
+			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
+				var result error
+				test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) {
+					if state.Snapshots != nil && state.StateDB != nil {
+						if _, err := state.Snapshots.Journal(state.StateDB.IntermediateRoot(false)); err != nil {
+							result = err
+							return
+						}
+					}
+					result = st.checkFailure(t, err)
 				})
-			}
+				return result
+			})
 		})
 	}
 }