Merge pull request #597 from gurukamath/valid_block_tests

Run all available `ValidBlock` tests
ethereum · Oct 17, 2022 · a04d7ea · a04d7ea
2 parents a7ea92e + 0103050
commit a04d7ea
Show file tree

Hide file tree

Showing 8 changed files with 170 additions and 110 deletions.
diff --git a/tests/berlin/test_state_transition.py b/tests/berlin/test_state_transition.py
@@ -27,7 +27,7 @@
 
 # Every test below takes more than  60s to run and
 # hence they've been marked as slow
-SLOW_TESTS = (
+GENERAL_STATE_SLOW_TESTS = (
     "stTimeConsuming/CALLBlake2f_MaxRounds.json",
     "stTimeConsuming/static_Call50000_sha256.json",
     "vmPerformance/loopExp.json",
@@ -61,7 +61,7 @@
     fetch_berlin_tests(
         test_dir,
         ignore_list=INCORRECT_UPSTREAM_STATE_TESTS,
-        slow_list=SLOW_TESTS,
+        slow_list=GENERAL_STATE_SLOW_TESTS,
         big_memory_list=BIG_MEMORY_TESTS,
     ),
     ids=idfn,
@@ -77,27 +77,33 @@ def test_general_state_tests(test_case: Dict) -> None:
 # Run legacy valid block tests
 test_dir = "tests/fixtures/BlockchainTests/ValidBlocks/"
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
-    "bcUncleTest/uncleHeaderAtBlock2.json",
-    "bcUncleSpecialTests/uncleBloomNot0.json",
-    "bcUncleSpecialTests/futureUncleTimestampDifficultyDrop.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest",
+    "bcTotalDifficultyTest",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+VALID_BLOCKS_SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_berlin_tests(test_dir, only_in=only_in),
+    fetch_berlin_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=VALID_BLOCKS_SLOW_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_berlin_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_berlin_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/byzantium/test_state_transition.py b/tests/byzantium/test_state_transition.py
@@ -63,24 +63,36 @@ def test_general_state_tests(test_case: Dict) -> None:
     "tests/fixtures/LegacyTests/Constantinople/BlockchainTests/ValidBlocks/"
 )
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest",
+    "bcTotalDifficultyTest",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
+BIG_MEMORY_TESTS = ("randomStatetest94_",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_byzantium_tests(test_dir, only_in=only_in),
+    fetch_byzantium_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=SLOW_TESTS,
+        big_memory_list=BIG_MEMORY_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_byzantium_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_byzantium_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/constantinople/test_state_transition.py b/tests/constantinople/test_state_transition.py
@@ -81,24 +81,36 @@ def test_general_state_tests(test_case: Dict) -> None:
     "tests/fixtures/LegacyTests/Constantinople/BlockchainTests/ValidBlocks/"
 )
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest",
+    "bcTotalDifficultyTest",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
+BIG_MEMORY_TESTS = ("randomStatetest94_",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_constantinople_tests(test_dir, only_in=only_in),
+    fetch_constantinople_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=SLOW_TESTS,
+        big_memory_list=BIG_MEMORY_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_constantinople_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_constantinople_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/frontier/test_state_transition.py b/tests/frontier/test_state_transition.py
@@ -48,27 +48,28 @@ def test_general_state_tests(test_case: Dict) -> None:
     "tests/fixtures/LegacyTests/Constantinople/BlockchainTests/ValidBlocks/"
 )
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
-    "bcUncleTest/uncleHeaderAtBlock2.json",
-    "bcUncleSpecialTests/uncleBloomNot0.json",
-    "bcUncleSpecialTests/futureUncleTimestampDifficultyDrop.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest/",
+    "bcTotalDifficultyTest/",
 )
 
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_frontier_tests(test_dir, only_in=only_in),
+    fetch_frontier_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_frontier_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_frontier_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/homestead/test_state_transition.py b/tests/homestead/test_state_transition.py
@@ -31,7 +31,7 @@
 
 # Every test below takes more than  60s to run and
 # hence they've been marked as slow
-SLOW_TESTS = (
+GENERAL_STATE_SLOW_TESTS = (
     "stRandom/randomStatetest177_d0g0v0.json",
     "stQuadraticComplexityTest/Call50000_d0g1v0.json",
     "stQuadraticComplexityTest/Call50000_sha256_d0g1v0.json",
@@ -116,7 +116,7 @@
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_homestead_tests(test_dir, slow_list=SLOW_TESTS),
+    fetch_homestead_tests(test_dir, slow_list=GENERAL_STATE_SLOW_TESTS),
     ids=idfn,
 )
 def test_general_state_tests(test_case: Dict) -> None:
@@ -132,26 +132,36 @@ def test_general_state_tests(test_case: Dict) -> None:
     "tests/fixtures/LegacyTests/Constantinople/BlockchainTests/ValidBlocks/"
 )
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
-    "bcUncleTest/uncleHeaderAtBlock2.json",
-    "bcUncleSpecialTests/uncleBloomNot0.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest/",
+    "bcTotalDifficultyTest/",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+VALID_BLOCKS_SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
+BIG_MEMORY_TESTS = ("randomStatetest94_",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_homestead_tests(test_dir, only_in=only_in),
+    fetch_homestead_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=VALID_BLOCKS_SLOW_TESTS,
+        big_memory_list=BIG_MEMORY_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_homestead_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_homestead_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/istanbul/test_state_transition.py b/tests/istanbul/test_state_transition.py
@@ -27,7 +27,7 @@
 
 # Every test below takes more than  60s to run and
 # hence they've been marked as slow
-SLOW_TESTS = (
+GENERAL_STATE_SLOW_TESTS = (
     "stTimeConsuming/CALLBlake2f_MaxRounds.json",
     "stTimeConsuming/static_Call50000_sha256.json",
     "vmPerformance/loopExp.json",
@@ -61,7 +61,7 @@
     fetch_istanbul_tests(
         test_dir,
         ignore_list=INCORRECT_UPSTREAM_STATE_TESTS,
-        slow_list=SLOW_TESTS,
+        slow_list=GENERAL_STATE_SLOW_TESTS,
         big_memory_list=BIG_MEMORY_TESTS,
     ),
     ids=idfn,
@@ -77,27 +77,33 @@ def test_general_state_tests(test_case: Dict) -> None:
 # Run legacy valid block tests
 test_dir = "tests/fixtures/BlockchainTests/ValidBlocks/"
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
-    "bcUncleTest/uncleHeaderAtBlock2.json",
-    "bcUncleSpecialTests/uncleBloomNot0.json",
-    "bcUncleSpecialTests/futureUncleTimestampDifficultyDrop.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest",
+    "bcTotalDifficultyTest",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+VALID_BLOCKS_SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_istanbul_tests(test_dir, only_in=only_in),
+    fetch_istanbul_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=VALID_BLOCKS_SLOW_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_istanbul_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_istanbul_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests

diff --git a/tests/spurious_dragon/test_state_transition.py b/tests/spurious_dragon/test_state_transition.py
@@ -48,27 +48,33 @@ def test_general_state_tests(test_case: Dict) -> None:
     "tests/fixtures/LegacyTests/Constantinople/BlockchainTests/ValidBlocks/"
 )
 
-only_in = (
-    "bcUncleTest/oneUncle.json",
-    "bcUncleTest/oneUncleGeneration2.json",
-    "bcUncleTest/oneUncleGeneration3.json",
-    "bcUncleTest/oneUncleGeneration4.json",
-    "bcUncleTest/oneUncleGeneration5.json",
-    "bcUncleTest/oneUncleGeneration6.json",
-    "bcUncleTest/twoUncle.json",
-    "bcUncleTest/uncleHeaderAtBlock2.json",
-    "bcUncleSpecialTests/uncleBloomNot0.json",
-    "bcUncleSpecialTests/futureUncleTimestampDifficultyDrop.json",
+IGNORE_LIST = (
+    "bcForkStressTest/ForkStressTest.json",
+    "bcGasPricerTest/RPC_API_Test.json",
+    "bcMultiChainTest",
+    "bcTotalDifficultyTest",
 )
 
+# Every test below takes more than  60s to run and
+# hence they've been marked as slow
+SLOW_TESTS = ("bcExploitTest/DelegateCallSpam.json",)
+
 
 @pytest.mark.parametrize(
     "test_case",
-    fetch_spurious_dragon_tests(test_dir, only_in=only_in),
+    fetch_spurious_dragon_tests(
+        test_dir,
+        ignore_list=IGNORE_LIST,
+        slow_list=SLOW_TESTS,
+    ),
     ids=idfn,
 )
-def test_uncles_correctness(test_case: Dict) -> None:
-    run_spurious_dragon_blockchain_st_tests(test_case)
+def test_valid_block_tests(test_case: Dict) -> None:
+    try:
+        run_spurious_dragon_blockchain_st_tests(test_case)
+    except KeyError:
+        # FIXME: Handle tests that don't have post state
+        pytest.xfail(f"{test_case} doesn't have post state")
 
 
 # Run legacy invalid block tests