Merge pull request #3927 from ethereum/fix-testgen-action

Fix test vector generators to work with Python 3.12
ethereum · Sep 18, 2024 · c8ba4d2 · c8ba4d2
2 parents 666b63a + 36da776
commit c8ba4d2
Show file tree

Hide file tree

Showing 24 changed files with 22 additions and 53 deletions.
diff --git a/.github/workflows/generate_vectors.yml b/.github/workflows/generate_vectors.yml
@@ -17,14 +17,14 @@ on:
 
 jobs:
   generate-tests:
-    runs-on: [self-hosted-ghr-custom, size-chungus-x64, profile-consensusSpecs]
+    runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           repository: 'ethereum/consensus-specs'
           path: 'consensus-specs'
-          ref: ${{ inputs.source_ref || 'dev' }}
+          ref: ${{ inputs.ref || 'dev' }}
       - name: Checkout consensus-spec-tests repository
         uses: actions/checkout@v4
         with:

diff --git a/Makefile b/Makefile
@@ -96,6 +96,9 @@ dist_check:
 dist_upload:
 	python3 -m twine upload dist/*
 
+build_wheel: install_test pyspec
+	. venv/bin/activate && \
+	python3 -m build --no-isolation --outdir ./dist ./
 
 # "make generate_tests" to run all generators
 generate_tests: $(GENERATOR_TARGETS)
@@ -195,7 +198,8 @@ define run_generator
 	cd $(GENERATOR_DIR)/$(1); \
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
-	pip3 install -r requirements.txt; \
+	pip3 install ../../../dist/eth2spec-*.whl; \
+	pip3 install 'eth2spec[generator]'; \
 	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
 	echo "generator $(1) finished"
 endef
@@ -217,7 +221,7 @@ gen_kzg_setups:
 
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-gen_%: $(TEST_VECTOR_DIR)
+gen_%: build_wheel $(TEST_VECTOR_DIR)
 	$(call run_generator,$*)
 
 detect_generator_incomplete: $(TEST_VECTOR_DIR)

diff --git a/requirements_preinstallation.txt b/requirements_preinstallation.txt
@@ -1,4 +1,5 @@
 pip>=24.0.0
 wheel>=0.44.0
 setuptools>=72.0.0
-pylint>=3.2.0
+pylint>=3.2.0
+build>=1.2.2
diff --git a/setup.py b/setup.py
@@ -524,26 +524,30 @@ def run(self):
     long_description=readme,
     long_description_content_type="text/markdown",
     author="ethereum",
-    url="https://github.com/ethereum/eth2.0-specs",
+    url="https://github.com/ethereum/consensus-specs",
     include_package_data=False,
-    package_data={'configs': ['*.yaml'],
-                  'presets': ['*.yaml'],
-                  'specs': ['**/*.md'],
-                  'eth2spec': ['VERSION.txt']},
+    package_data={
+        'configs': ['*.yaml'],
+        'eth2spec': ['VERSION.txt'],
+        'presets': ['**/*.yaml', '**/*.json'],
+        'specs': ['**/*.md'],
+        'sync': ['optimistic.md'],
+    },
     package_dir={
-        "eth2spec": "tests/core/pyspec/eth2spec",
         "configs": "configs",
+        "eth2spec": "tests/core/pyspec/eth2spec",
         "presets": "presets",
         "specs": "specs",
+        "sync": "sync",
     },
-    packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'],
+    packages=find_packages(where='tests/core/pyspec') + ['configs', 'presets', 'specs', 'presets', 'sync'],
     py_modules=["eth2spec"],
     cmdclass=commands,
     python_requires=">=3.9, <4",
     extras_require={
         "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
         "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"],
-        "generator": ["python-snappy==0.6.1", "filelock", "pathos==0.3.0"],
+        "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"],
         "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3",  "mkdocs-awesome-pages-plugin==2.8.0"]
     },
     install_requires=[

diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt
diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt
diff --git a/tests/generators/finality/requirements.txt b/tests/generators/finality/requirements.txt
diff --git a/tests/generators/fork_choice/requirements.txt b/tests/generators/fork_choice/requirements.txt
diff --git a/tests/generators/forks/requirements.txt b/tests/generators/forks/requirements.txt
diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt
diff --git a/tests/generators/kzg_4844/requirements.txt b/tests/generators/kzg_4844/requirements.txt
diff --git a/tests/generators/kzg_7594/requirements.txt b/tests/generators/kzg_7594/requirements.txt
diff --git a/tests/generators/light_client/requirements.txt b/tests/generators/light_client/requirements.txt
diff --git a/tests/generators/merkle_proof/requirements.txt b/tests/generators/merkle_proof/requirements.txt
diff --git a/tests/generators/networking/requirements.txt b/tests/generators/networking/requirements.txt
diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt
diff --git a/tests/generators/random/requirements.txt b/tests/generators/random/requirements.txt
diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt
diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt
diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt
diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt
diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt
diff --git a/tests/generators/sync/requirements.txt b/tests/generators/sync/requirements.txt
diff --git a/tests/generators/transition/requirements.txt b/tests/generators/transition/requirements.txt