Skip to content

Commit 0e7c0f8

Browse files
winsvegaspencer-tbdanceratopz
authored
test(fw): Unit tests for post state verification errors (#350)
* test(fw): Unit tests for post state verification. * docs: Add changelog entry. * refactor: make post state verification tests more pytesty (#393) * refactor: use pytest fixtures as apposed to helper function * refactor: update comments due to state test fixtures #356 The functionality from ethereum_test.filler.fill_test() got moved to the StateTest, respectively BlockchainTest, generate() method. --------- Co-authored-by: spencer-tb <spencer@spencertaylorbrown.uk> Co-authored-by: danceratopz <danceratopz@gmail.com>
1 parent c4317dd commit 0e7c0f8

15 files changed

+255
-13
lines changed

‎docs/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ Test fixtures for use by clients are available for each release on the [Github r
5151
- ✨ Add a helper class `ethereum_test_tools.TestParameterGroup` to define test parameters as dataclasses and auto-generate test IDs ([#364](https://github.com/ethereum/execution-spec-tests/pull/364)).
5252
- 🔀 Change custom exception classes to dataclasses to improve testability ([#386](https://github.com/ethereum/execution-spec-tests/pull/386)).
5353
- 🔀 Updates fork name from Merge to Paris ([#363](https://github.com/ethereum/execution-spec-tests/pull/363)).
54+
- ✨ Add framework unit tests for post state exception verification ([#350](https://github.com/ethereum/execution-spec-tests/pull/350)).
5455

5556
### 🔧 EVM Tools
5657

‎src/ethereum_test_tools/common/types.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -500,7 +500,7 @@ def must_contain(self, address: str, other: "Storage"):
500500
raise Storage.MissingKey(key=key)
501501
elif self.data[key] != other.data[key]:
502502
raise Storage.KeyValueMismatch(
503-
address=address, key=key, got=self.data[key], want=other.data[key]
503+
address=address, key=key, want=self.data[key], got=other.data[key]
504504
)
505505

506506
def must_be_equal(self, address: str, other: "Storage"):
@@ -511,7 +511,7 @@ def must_be_equal(self, address: str, other: "Storage"):
511511
for key in self.data.keys() & other.data.keys():
512512
if self.data[key] != other.data[key]:
513513
raise Storage.KeyValueMismatch(
514-
address=address, key=key, got=self.data[key], want=other.data[key]
514+
address=address, key=key, want=self.data[key], got=other.data[key]
515515
)
516516

517517
# Test keys contained in either one of the storage objects

‎src/ethereum_test_tools/tests/test_filling/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"""
2+
`ethereum_test_tools.filling` verification tests.
3+
"""

‎src/ethereum_test_tools/tests/test_fixtures/blockchain_london_invalid_filled.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_invalid_filled.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/blockchain_london_valid_filled.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_valid_filled.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_invalid_filled_hive.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_invalid_filled_hive.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_valid_filled_hive.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_valid_filled_hive.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_istanbul_blockchain_test.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_istanbul_blockchain_test.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_london_blockchain_test.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_london_blockchain_test.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_paris_blockchain_test_hive.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_blockchain_test_hive.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_paris_state_test.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_state_test.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_blockchain_test_hive.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_blockchain_test_hive.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_state_test.json renamed to ‎src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_state_test.json

File renamed without changes.

‎src/ethereum_test_tools/tests/test_filling/test_expect.py

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
"""
2+
Test fixture post state (expect section) during state fixture generation.
3+
"""
4+
from typing import Any, Mapping
5+
6+
import pytest
7+
8+
from ethereum_test_forks import Fork, get_deployed_forks
9+
from evm_transition_tool import FixtureFormats, GethTransitionTool
10+
11+
from ...common import Account, Environment, Transaction, to_address
12+
from ...common.types import Storage
13+
from ...spec import StateTest
14+
15+
ADDRESS_UNDER_TEST = to_address(0x01)
16+
17+
18+
@pytest.fixture
19+
def pre(request) -> Mapping[Any, Any]:
20+
"""
21+
The pre state: Set from the test's indirectly parametrized `pre` parameter.
22+
"""
23+
return request.param
24+
25+
26+
@pytest.fixture
27+
def post(request) -> Mapping[Any, Any]: # noqa: D103
28+
"""
29+
The post state: Set from the test's indirectly parametrized `post` parameter.
30+
"""
31+
return request.param
32+
33+
34+
@pytest.fixture
35+
def fork() -> Fork: # noqa: D103
36+
return get_deployed_forks()[-1]
37+
38+
39+
@pytest.fixture
40+
def state_test( # noqa: D103
41+
fork: Fork, pre: Mapping[Any, Any], post: Mapping[Any, Any]
42+
) -> StateTest:
43+
return StateTest(
44+
env=Environment(),
45+
pre=pre,
46+
post=post,
47+
tx=Transaction(),
48+
tag="post_value_mismatch",
49+
fixture_format=FixtureFormats.STATE_TEST,
50+
)
51+
52+
53+
@pytest.fixture
54+
def t8n() -> GethTransitionTool: # noqa: D103
55+
return GethTransitionTool()
56+
57+
58+
# Storage value mismatch tests
59+
@pytest.mark.parametrize(
60+
"pre,post,expected_exception",
61+
[
62+
( # mismatch_1: 1:1 vs 1:2
63+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)},
64+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})},
65+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=1),
66+
),
67+
( # mismatch_2: 1:1 vs 2:1
68+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)},
69+
{ADDRESS_UNDER_TEST: Account(storage={"0x02": "0x01"})},
70+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1),
71+
),
72+
( # mismatch_2_a: 1:1 vs 0:0
73+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)},
74+
{ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x00"})},
75+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1),
76+
),
77+
( # mismatch_2_b: 1:1 vs empty
78+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)},
79+
{ADDRESS_UNDER_TEST: Account(storage={})},
80+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1),
81+
),
82+
( # mismatch_3: 0:0 vs 1:2
83+
{ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x00"}, nonce=1)},
84+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})},
85+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=0),
86+
),
87+
( # mismatch_3_a: empty vs 1:2
88+
{ADDRESS_UNDER_TEST: Account(storage={}, nonce=1)},
89+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})},
90+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=0),
91+
),
92+
( # mismatch_4: 0:3, 1:2 vs 1:2
93+
{ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x03", "0x01": "0x02"}, nonce=1)},
94+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})},
95+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=0, want=0, got=3),
96+
),
97+
( # mismatch_5: 1:2, 2:3 vs 1:2
98+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02", "0x02": "0x03"}, nonce=1)},
99+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})},
100+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=2, want=0, got=3),
101+
),
102+
( # mismatch_6: 1:2 vs 1:2, 2:3
103+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"}, nonce=1)},
104+
{ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02", "0x02": "0x03"})},
105+
Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=2, want=3, got=0),
106+
),
107+
],
108+
)
109+
def test_post_storage_value_mismatch(pre, post, expected_exception, state_test, t8n, fork):
110+
"""
111+
Test post state `Account.storage` exceptions during state test fixture generation.
112+
"""
113+
with pytest.raises(Storage.KeyValueMismatch) as e_info:
114+
state_test.generate(t8n=t8n, fork=fork)
115+
assert e_info.value == expected_exception
116+
117+
118+
# Nonce value mismatch tests
119+
@pytest.mark.parametrize(
120+
"pre,post",
121+
[
122+
({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=2)}),
123+
({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=0)}),
124+
({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=None)}),
125+
],
126+
)
127+
def test_post_nonce_value_mismatch(pre, post, state_test, t8n, fork):
128+
"""
129+
Test post state `Account.nonce` verification and exceptions during state test
130+
fixture generation.
131+
"""
132+
pre_nonce = pre[ADDRESS_UNDER_TEST].nonce
133+
post_nonce = post[ADDRESS_UNDER_TEST].nonce
134+
if post_nonce is None: # no exception
135+
state_test.generate(t8n=t8n, fork=fork)
136+
return
137+
with pytest.raises(Account.NonceMismatch) as e_info:
138+
state_test.generate(t8n=t8n, fork=fork)
139+
assert e_info.value == Account.NonceMismatch(
140+
address=ADDRESS_UNDER_TEST, want=post_nonce, got=pre_nonce
141+
)
142+
143+
144+
# Code value mismatch tests
145+
@pytest.mark.parametrize(
146+
"pre,post",
147+
[
148+
({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code="0x01")}),
149+
({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code="0x")}),
150+
({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code=None)}),
151+
],
152+
indirect=["pre", "post"],
153+
)
154+
def test_post_code_value_mismatch(pre, post, state_test, t8n, fork):
155+
"""
156+
Test post state `Account.code` verification and exceptions during state test
157+
fixture generation.
158+
"""
159+
pre_code = pre[ADDRESS_UNDER_TEST].code
160+
post_code = post[ADDRESS_UNDER_TEST].code
161+
if post_code is None: # no exception
162+
state_test.generate(t8n=t8n, fork=fork)
163+
return
164+
with pytest.raises(Account.CodeMismatch) as e_info:
165+
state_test.generate(t8n=t8n, fork=fork)
166+
assert e_info.value == Account.CodeMismatch(
167+
address=ADDRESS_UNDER_TEST, want=post_code, got=pre_code
168+
)
169+
170+
171+
# Balance value mismatch tests
172+
@pytest.mark.parametrize(
173+
"pre,post",
174+
[
175+
({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=2)}),
176+
({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=0)}),
177+
({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=None)}),
178+
],
179+
indirect=["pre", "post"],
180+
)
181+
def test_post_balance_value_mismatch(pre, post, state_test, t8n, fork):
182+
"""
183+
Test post state `Account.balance` verification and exceptions during state test
184+
fixture generation.
185+
"""
186+
pre_balance = pre[ADDRESS_UNDER_TEST].balance
187+
post_balance = post[ADDRESS_UNDER_TEST].balance
188+
if post_balance is None: # no exception
189+
state_test.generate(t8n=t8n, fork=fork)
190+
return
191+
with pytest.raises(Account.BalanceMismatch) as e_info:
192+
state_test.generate(t8n=t8n, fork=fork)
193+
assert e_info.value == Account.BalanceMismatch(
194+
address=ADDRESS_UNDER_TEST, want=post_balance, got=pre_balance
195+
)
196+
197+
198+
# Account mismatch tests
199+
@pytest.mark.parametrize(
200+
"pre,post,error_str",
201+
[
202+
(
203+
{ADDRESS_UNDER_TEST: Account(balance=1)},
204+
{ADDRESS_UNDER_TEST: Account()},
205+
None,
206+
),
207+
(
208+
{ADDRESS_UNDER_TEST: Account(balance=1)},
209+
{ADDRESS_UNDER_TEST: Account(balance=1), to_address(0x02): Account(balance=1)},
210+
"expected account not found",
211+
),
212+
(
213+
{ADDRESS_UNDER_TEST: Account(balance=1)},
214+
{},
215+
None,
216+
),
217+
(
218+
{ADDRESS_UNDER_TEST: Account(balance=1)},
219+
{ADDRESS_UNDER_TEST: Account.NONEXISTENT},
220+
"found unexpected account",
221+
),
222+
],
223+
indirect=["pre", "post"],
224+
)
225+
def test_post_account_mismatch(state_test, t8n, fork, error_str):
226+
"""
227+
Test post state `Account` verification and exceptions during state test
228+
fixture generation.
229+
"""
230+
if error_str is None:
231+
state_test.generate(t8n=t8n, fork=fork)
232+
return
233+
with pytest.raises(Exception) as e_info:
234+
state_test.generate(t8n=t8n, fork=fork)
235+
assert error_str in str(e_info.value)

‎src/ethereum_test_tools/tests/test_filler.py renamed to ‎src/ethereum_test_tools/tests/test_filling/test_fixtures.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
2-
Test suite for `ethereum_test` module.
2+
Test suite for `ethereum_test_tools.filling` fixture generation.
33
"""
44

55
import json
@@ -12,13 +12,13 @@
1212
from ethereum_test_forks import Berlin, Fork, Istanbul, London, Paris, Shanghai
1313
from evm_transition_tool import FixtureFormats, GethTransitionTool
1414

15-
from ..code import Yul
16-
from ..common import Account, Environment, TestAddress, Transaction, to_json
17-
from ..spec import BlockchainTest, StateTest
18-
from ..spec.blockchain.types import Block
19-
from ..spec.blockchain.types import Fixture as BlockchainFixture
20-
from ..spec.blockchain.types import FixtureCommon as BlockchainFixtureCommon
21-
from .conftest import SOLC_PADDING_VERSION
15+
from ...code import Yul
16+
from ...common import Account, Environment, TestAddress, Transaction, to_json
17+
from ...spec import BlockchainTest, StateTest
18+
from ...spec.blockchain.types import Block
19+
from ...spec.blockchain.types import Fixture as BlockchainFixture
20+
from ...spec.blockchain.types import FixtureCommon as BlockchainFixtureCommon
21+
from ..conftest import SOLC_PADDING_VERSION
2222

2323

2424
def remove_info(fixture_json: Dict[str, Any]): # noqa: D103
@@ -161,7 +161,8 @@ def test_fill_state_test(
161161
"src",
162162
"ethereum_test_tools",
163163
"tests",
164-
"test_fixtures",
164+
"test_filling",
165+
"fixtures",
165166
expected_json_file,
166167
)
167168
) as f:
@@ -461,7 +462,8 @@ def test_fill_blockchain_valid_txs(
461462
"src",
462463
"ethereum_test_tools",
463464
"tests",
464-
"test_fixtures",
465+
"test_filling",
466+
"fixtures",
465467
expected_json_file,
466468
)
467469
) as f:
@@ -810,7 +812,8 @@ def test_fill_blockchain_invalid_txs(
810812
"src",
811813
"ethereum_test_tools",
812814
"tests",
813-
"test_fixtures",
815+
"test_filling",
816+
"fixtures",
814817
expected_json_file,
815818
)
816819
) as f:

0 commit comments

Comments
 (0)