update test generators

Makefile

@@ -167,7 +167,7 @@ define run_generator
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
 	pip3 install -r requirements.txt; \
-	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
+	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
 	echo "generator $(1) finished"
 endef
 

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -34,18 +34,6 @@ def validate_output_dir(path_str):
     return path
 
 
-def validate_configs_dir(path_str):
-    path = Path(path_str)
-
-    if not path.exists():
-        raise argparse.ArgumentTypeError("Configs directory must exist")
-
-    if not path.is_dir():
-        raise argparse.ArgumentTypeError("Config path must lead to a directory")
-
-    return path
-
-
 def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     """
     Implementation for a general test generator.
@@ -76,22 +64,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
         default=False,
         help="if set re-generate and overwrite test files if they already exist",
     )
-    parser.add_argument(
-        "-c",
-        "--configs-path",
-        dest="configs_path",
-        required=True,
-        type=validate_configs_dir,
-        help="specify the path of the configs directory",
-    )
     parser.add_argument(
         "-l",
-        "--config-list",
-        dest="config_list",
+        "--preset-list",
+        dest="preset_list",
         nargs='*',
         type=str,
         required=False,
-        help="specify configs to run with. Allows all if no config names are specified.",
+        help="specify presets to run with. Allows all if no preset names are specified.",
     )
 
     args = parser.parse_args()
@@ -107,27 +87,22 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     log_file = Path(output_dir) / 'testgen_error_log.txt'
 
     print(f"Generating tests into {output_dir}")
-    print(f"Reading configs from {args.configs_path}")
     print(f'Error log file: {log_file}')
 
-    configs = args.config_list
-    if configs is None:
-        configs = []
+    presets = args.preset_list
+    if presets is None:
+        presets = []
 
-    if len(configs) != 0:
-        print(f"Filtering test-generator runs to only include configs: {', '.join(configs)}")
+    if len(presets) != 0:
+        print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}")
 
     for tprov in test_providers:
-        # loads configuration etc.
-        config_name = tprov.prepare(args.configs_path)
-        if len(configs) != 0 and config_name not in configs:
-            print(f"skipping tests with config '{config_name}' since it is filtered out")
-            continue
+        # runs anything that we don't want to repeat for every test case.
+        tprov.prepare()
 
-        print(f"generating tests with config '{config_name}' ...")
         for test_case in tprov.make_cases():
             case_dir = (
-                Path(output_dir) / Path(config_name) / Path(test_case.fork_name)
+                Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name)
                 / Path(test_case.runner_name) / Path(test_case.handler_name)
                 / Path(test_case.suite_name) / Path(test_case.case_name)
             )

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py

@@ -19,6 +19,7 @@
 @dataclass
 class TestCase(object):
     fork_name: str
+    preset_name: str
     runner_name: str
     handler_name: str
     suite_name: str
@@ -28,8 +29,7 @@ class TestCase(object):
 
 @dataclass
 class TestProvider(object):
-    # Prepares the context with a configuration, loaded from the given config path.
-    # fn(config path) => chosen config name
-    prepare: Callable[[str], str]
+    # Prepares the context for the provider as a whole, as opposed to per-test-case changes.
+    prepare: Callable[[], None]
     # Retrieves an iterable of cases, called after prepare()
     make_cases: Callable[[], Iterable[TestCase]]

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -1,18 +1,18 @@
-from importlib import reload, import_module
+from importlib import import_module
 from inspect import getmembers, isfunction
 from typing import Any, Callable, Dict, Iterable, Optional
 
-from eth2spec.config import config_util
 from eth2spec.utils import bls
-from eth2spec.test.helpers.constants import ALL_CONFIGS, TESTGEN_FORKS
-from eth2spec.test.helpers.typing import SpecForkName, ConfigName
+from eth2spec.test.helpers.constants import ALL_PRESETS, TESTGEN_FORKS
+from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
 
 from eth2spec.gen_helpers.gen_base import gen_runner
 from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
 
 
 def generate_from_tests(runner_name: str, handler_name: str, src: Any,
-                        fork_name: SpecForkName, bls_active: bool = True,
+                        fork_name: SpecForkName, preset_name: PresetBaseName,
+                        bls_active: bool = True,
                         phase: Optional[str]=None) -> Iterable[TestCase]:
     """
     Generate a list of test cases by running tests from the given src in generator-mode.
@@ -21,8 +21,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
     :param src: to retrieve tests from (discovered using inspect.getmembers).
     :param fork_name: the folder name for these tests.
            (if multiple forks are applicable, indicate the last fork)
+    :param preset_name: to select a preset. Tests that do not support the preset will be skipped.
     :param bls_active: optional, to override BLS switch preference. Defaults to True.
     :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value.
+           Set to the pre-fork (w.r.t. fork_name) in multi-fork tests.
     :return: an iterable of test cases.
     """
     fn_names = [
@@ -44,63 +46,63 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
 
         yield TestCase(
             fork_name=fork_name,
+            preset_name=preset_name,
             runner_name=runner_name,
             handler_name=handler_name,
             suite_name='pyspec_tests',
             case_name=case_name,
             # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
-            case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active)
+            case_fn=lambda: tfn(generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active)
         )
 
 
-def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider],
-                 config_name: ConfigName,
+def get_provider(create_provider_fn: Callable[[SpecForkName, PresetBaseName, str, str], TestProvider],
                  fork_name: SpecForkName,
+                 preset_name: PresetBaseName,
                  all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]:
     for key, mod_name in all_mods[fork_name].items():
         yield create_provider_fn(
             fork_name=fork_name,
+            preset_name=preset_name,
             handler_name=key,
             tests_src_mod_name=mod_name,
-            config_name=config_name,
         )
 
 
-def get_create_provider_fn(
-        runner_name: str, config_name: ConfigName, specs: Iterable[Any]
-) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]:
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        for spec in specs:
-            reload(spec)
+def get_create_provider_fn(runner_name: str) -> Callable[[SpecForkName, str, str, PresetBaseName], TestProvider]:
+    def prepare_fn() -> None:
         bls.use_milagro()
-        return config_name
+        return
 
-    def create_provider(fork_name: SpecForkName, handler_name: str,
-                        tests_src_mod_name: str, config_name: ConfigName) -> TestProvider:
+    def create_provider(fork_name: SpecForkName, preset_name: PresetBaseName,
+                        handler_name: str, tests_src_mod_name: str) -> TestProvider:
         def cases_fn() -> Iterable[TestCase]:
             tests_src = import_module(tests_src_mod_name)
             return generate_from_tests(
                 runner_name=runner_name,
                 handler_name=handler_name,
                 src=tests_src,
                 fork_name=fork_name,
+                preset_name=preset_name,
             )
 
         return TestProvider(prepare=prepare_fn, make_cases=cases_fn)
     return create_provider
 
 
-def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None:
+def run_state_test_generators(runner_name: str,
+                              all_mods: Dict[str, Dict[str, str]],
+                              presets: Iterable[PresetBaseName] = ALL_PRESETS,
+                              forks: Iterable[SpecForkName] = TESTGEN_FORKS) -> None:
     """
-    Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner.
+    Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_PRESETS` presets of the given runner.
     """
-    for config_name in ALL_CONFIGS:
-        for fork_name in TESTGEN_FORKS:
+    for preset_name in presets:
+        for fork_name in forks:
             if fork_name in all_mods:
                 gen_runner.run_generator(runner_name, get_provider(
-                    create_provider_fn=get_create_provider_fn(runner_name, config_name, specs),
-                    config_name=config_name,
+                    create_provider_fn=get_create_provider_fn(runner_name),
                     fork_name=fork_name,
+                    preset_name=preset_name,
                     all_mods=all_mods,
                 ))

tests/core/pyspec/eth2spec/test/context.py

@@ -414,7 +414,7 @@ def wrapper(*args, spec: Spec, **kw):
             test_config = {k: config_types[k](v) for k, v in tmp_config.items()}
 
             # Output the config for test vectors  (TODO: check config YAML encoding)
-            yield 'config', test_config
+            yield 'config', 'data', test_config
 
             spec.config = spec.Configuration(**test_config)
 

tests/core/pyspec/eth2spec/test/helpers/constants.py

@@ -31,4 +31,4 @@
 MAINNET = PresetBaseName('mainnet')
 MINIMAL = PresetBaseName('minimal')
 
-ALL_CONFIGS = (MINIMAL, MAINNET)
+ALL_PRESETS = (MINIMAL, MAINNET)

tests/generators/bls/main.py

@@ -358,17 +358,18 @@ def case05_aggregate_verify():
 def create_provider(handler_name: str,
                     test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider:
 
-    def prepare_fn(configs_path: str) -> str:
+    def prepare_fn() -> None:
         # Nothing to load / change in spec. Maybe in future forks.
         # Put the tests into the general config category, to not require any particular configuration.
-        return 'general'
+        return
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
         for data in test_case_fn():
             print(data)
             (case_name, case_content) = data
             yield gen_typing.TestCase(
                 fork_name=PHASE0,
+                preset_name='general',
                 runner_name='bls',
                 handler_name=handler_name,
                 suite_name='small',

tests/generators/epoch_processing/main.py

@@ -1,13 +1,7 @@
 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
-from eth2spec.merge import spec as spec_merge
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
 
 
-specs = (spec_phase0, spec_altair, spec_merge)
-
-
 if __name__ == "__main__":
     phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [
         'justification_and_finalization',
@@ -45,4 +39,4 @@
         MERGE: merge_mods,
     }
 
-    run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods)
+    run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods)

tests/generators/finality/main.py

@@ -1,13 +1,7 @@
 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
-from eth2spec.merge import spec as spec_merge
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
 
 
-specs = (spec_phase0, spec_altair, spec_merge)
-
-
 if __name__ == "__main__":
     phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'}
     altair_mods = phase_0_mods   # No additional Altair specific finality tests
@@ -19,4 +13,4 @@
         MERGE: spec_merge,
     }
 
-    run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods)
+    run_state_test_generators(runner_name="finality", all_mods=all_mods)

tests/generators/fork_choice/main.py

@@ -1,13 +1,7 @@
 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
-from eth2spec.merge import spec as spec_merge
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
 
 
-specs = (spec_phase0, spec_altair, spec_merge)
-
-
 if __name__ == "__main__":
     phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [
         'get_head',
@@ -23,4 +17,4 @@
         MERGE: merge_mods,
     }
 
-    run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods)
+    run_state_test_generators(runner_name="fork_choice", all_mods=all_mods)

tests/generators/forks/main.py

@@ -1,30 +1,27 @@
-from importlib import reload
 from typing import Iterable
 
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MINIMAL, MAINNET
-from eth2spec.config import config_util
-from eth2spec.test.altair.fork import test_fork as test_altair_forks
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
+from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName
+from eth2spec.test.altair.fork import test_altair_fork_basic
+from eth2spec.test.altair.fork import test_altair_fork_random
 
 from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing
 from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests
 
 
-def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider:
+def create_provider(tests_src, preset_name: PresetBaseName,
+                    phase: SpecForkName, fork_name: SpecForkName) -> gen_typing.TestProvider:
 
-    def prepare_fn(configs_path: str) -> str:
-        config_util.prepare_config(configs_path, config_name)
-        reload(spec_phase0)
-        reload(spec_altair)
-        return config_name
+    def prepare_fn() -> None:
+        return
 
     def cases_fn() -> Iterable[gen_typing.TestCase]:
         return generate_from_tests(
             runner_name='fork',
             handler_name='fork',
             src=tests_src,
             fork_name=fork_name,
+            preset_name=preset_name,
             phase=phase,
         )
 
@@ -33,6 +30,8 @@ def cases_fn() -> Iterable[gen_typing.TestCase]:
 
 if __name__ == "__main__":
     gen_runner.run_generator("forks", [
-        create_provider(test_altair_forks, MINIMAL, PHASE0, ALTAIR),
-        create_provider(test_altair_forks, MAINNET, PHASE0, ALTAIR),
+        create_provider(test_altair_fork_basic, MINIMAL, PHASE0, ALTAIR),
+        create_provider(test_altair_fork_basic, MAINNET, PHASE0, ALTAIR),
+        create_provider(test_altair_fork_random, MINIMAL, PHASE0, ALTAIR),
+        create_provider(test_altair_fork_random, MAINNET, PHASE0, ALTAIR),
     ])

tests/generators/genesis/main.py

@@ -1,12 +1,7 @@
 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
-from eth2spec.phase0 import spec as spec_phase0
-from eth2spec.altair import spec as spec_altair
 from eth2spec.test.helpers.constants import PHASE0, ALTAIR
 
 
-specs = (spec_phase0, spec_altair)
-
-
 if __name__ == "__main__":
     phase_0_mods = {key: 'eth2spec.test.phase0.genesis.test_' + key for key in [
         'initialization',
@@ -18,4 +13,4 @@
         ALTAIR: altair_mods,
     }
 
-    run_state_test_generators(runner_name="genesis", specs=specs, all_mods=all_mods)
+    run_state_test_generators(runner_name="genesis", all_mods=all_mods)