From 98cf01d22dbb1ce2deada5dd11a5d029e54cf6a4 Mon Sep 17 00:00:00 2001
From: Justin Traglia <95511699+jtraglia@users.noreply.github.com>
Date: Fri, 13 Dec 2024 13:30:23 -0600
Subject: [PATCH] Refactor the main Makefile (#3988)

---
 .circleci/config.yml                   |  70 ++--
 .github/workflows/docs.yml             |   2 +-
 .github/workflows/generate_vectors.yml |  11 +-
 .github/workflows/run-tests.yml        |  55 +--
 .gitignore                             |   1 +
 Makefile                               | 449 ++++++++++++++-----------
 README.md                              |   3 +-
 docker/Dockerfile                      |   5 +-
 docker/README.md                       |   2 +-
 scripts/build_run_docker_tests.sh      |   6 +-
 setup.py                               |   2 +-
 tests/README.md                        |  56 +--
 tests/core/pyspec/README.md            |  75 +----
 tests/generators/README.md             |  33 +-
 14 files changed, 331 insertions(+), 439 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 80d44bef37..1de55179d4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,19 +35,19 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v25-pyspec
+          venv_name: v30-pyspec
           reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v25-pyspec
+          venv_name: v30-pyspec
           reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
           venv_path: ./venv
 jobs:
   checkout_specs:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       # Restore git repo at point close to target branch/revision, to speed up checkout
@@ -67,7 +67,7 @@ jobs:
             - ~/specs-repo
   install_pyspec_test:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -75,11 +75,11 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
-          command: make install_test
+          command: make eth2spec
       - save_pyspec_cached_venv
   test-phase0:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -87,12 +87,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=phase0
+          command: make test fork=phase0
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-altair:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -100,12 +100,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=altair
+          command: make test fork=altair
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-bellatrix:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -113,12 +113,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=bellatrix
+          command: make test fork=bellatrix
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-capella:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -126,12 +126,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=capella
+          command: make test fork=capella
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-deneb:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -139,12 +139,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=deneb
+          command: make test fork=deneb
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-electra:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -152,12 +152,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=electra
+          command: make test fork=electra
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-fulu:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -165,12 +165,12 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=fulu
+          command: make test fork=fulu
       - store_test_results:
           path: tests/core/pyspec/test-reports
   test-whisk:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -178,41 +178,23 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Run py-tests
-          command: make citest fork=whisk
+          command: make test fork=whisk
       - store_test_results:
           path: tests/core/pyspec/test-reports
-  table_of_contents:
-    docker:
-      - image: circleci/node:10.16.3
-    working_directory: ~/specs-repo
-    steps:
-      - checkout
-      - run:
-          name: Check table of contents
-          command: sudo npm install -g doctoc@2.2.0 && make check_toc
-  codespell:
-    docker:
-      - image: cimg/python:3.12.4
-    working_directory: ~/specs-repo
-    steps:
-      - checkout
-      - run:
-          name: Check codespell
-          command: pip install 'codespell<3.0.0,>=2.0.0' --user &&  make codespell
   lint:
     docker:
-      - image: cimg/python:3.12.4
+      - image: cimg/python:3.12-node
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
+      - run:
+          name: Install doctoc
+          command: sudo npm install -g doctoc@2.2.0
       - run:
           name: Run linter for pyspec
           command: make lint
-      - run:
-          name: Run linter for test generators
-          command: make lint_generators
 workflows:
   version: 2.1
   test_spec:
@@ -245,8 +227,6 @@ workflows:
       - test-whisk:
           requires:
             - install_pyspec_test
-      - table_of_contents
-      - codespell
       - lint:
           requires:
             - install_pyspec_test
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index b7bfb3538a..930781e77c 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Build docs
-        run: make copy_docs
+        run: make _copy_docs
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x
diff --git a/.github/workflows/generate_vectors.yml b/.github/workflows/generate_vectors.yml
index 1f14ff9158..128d402776 100644
--- a/.github/workflows/generate_vectors.yml
+++ b/.github/workflows/generate_vectors.yml
@@ -36,19 +36,10 @@ jobs:
         with:
           python-version: '3.12.4'
           cache: ''
-      - name: Clean up Spec Repository
-        run: |
-          cd consensus-specs
-          make clean
-      - name: Install dependencies and generate pyspec
-        run: |
-          cd consensus-specs
-          make install_test
-          make -B pyspec
       - name: Generate tests
         run: |
           cd consensus-specs
-          make -j 16 generate_tests 2>&1 | tee ../consensustestgen.log
+          make -j 16 gen_all 2>&1 | tee ../consensustestgen.log
           cp -r presets/ ../consensus-spec-tests/presets
           cp -r configs/ ../consensus-spec-tests/configs
           find . -type d -empty -delete
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 84c5d9cee8..3f10d19668 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -24,9 +24,9 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  table_of_contents:
-   runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
-   steps:
+  lint:
+    runs-on: [self-hosted-ghr-custom, size-l-x64, profile-consensusSpecs]
+    steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Setup Node.js
@@ -34,40 +34,15 @@ jobs:
         with:
           node-version: '20'
           cache: ''
-      - name: Check table of contents
-        run: npm install -g doctoc@2.2.0 && make check_toc
-
-  codespell:
-   runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
-   steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12.4'
-          cache: ''
-      - name: Check codespell
-        run: make codespell
-
-  lint:
-    runs-on: [self-hosted-ghr-custom, size-l-x64, profile-consensusSpecs]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Setup Rust for dependencies
-        uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12.4'
           cache: ''
-      - name: Install pyspec requirements
-        run: make install_test
+      - name: Install doctoc
+        run: npm install -g doctoc@2.2.0
       - name: Run linter for pyspec
         run: make lint
-      - name: Run linter for test generators
-        run: make lint_generators
 
   whitespace:
     runs-on: [self-hosted-ghr-custom, size-l-x64, profile-consensusSpecs]
@@ -83,7 +58,7 @@ jobs:
 
   pyspec-tests:
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
-    needs: [lint,codespell,table_of_contents]
+    needs: [lint]
     strategy:
       matrix:
         version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "fulu", "whisk"]
@@ -97,26 +72,24 @@ jobs:
         with:
           python-version: '3.12.4'
           cache: ''
-      - name: set TEST_PRESET_TYPE
+      - name: set preset
         if: github.event.inputs.test_preset_type != ''
         run: |
           echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
-      - name: set TEST_PRESET_TYPE
+      - name: set preset
         if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }}
         run: |
-          echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE}}" >> $GITHUB_ENV
-      - name: set TEST_PRESET_TYPE
+          echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
+      - name: set preset
         if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
         run: |
           echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
-      - name: set TEST_PRESET_TYPE
+      - name: set preset
         if: github.event.schedule=='0 0 * * *'
         run: |
           echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
-      - name: Install pyspec requirements
-        run: make install_test
       - name: test-${{ matrix.version }}
-        run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}}
+        run: make test fork=${{ matrix.version }} preset=${{ env.spec_test_preset_type }}
       - uses: actions/upload-artifact@v4
         if: always()
         with:
@@ -133,10 +106,8 @@ jobs:
         with:
           python-version: '3.12.4'
           cache: ''
-      - name: Install pyspec requirements
-        run: make install_test
       - name: Run generators with --modcheck
-        run: make generate_tests modcheck=true 2>&1 | tee consensustestgen.log
+        run: make gen_all modcheck=true 2>&1 | tee consensustestgen.log
       - name: Check for errors
         run: |
           if grep -q "\[ERROR\]" consensustestgen.log; then
diff --git a/.gitignore b/.gitignore
index 57cd180030..56f84dafd8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ venv
 .venv
 /.pytest_cache
 *.swp
+.eth2spec
 
 build/
 output/
diff --git a/Makefile b/Makefile
index 48b1b4fbf6..a3a3e24288 100644
--- a/Makefile
+++ b/Makefile
@@ -1,134 +1,198 @@
-SPEC_DIR = ./specs
-SSZ_DIR = ./ssz
-TEST_LIBS_DIR = ./tests/core
-TEST_GENERATORS_DIR = ./tests/generators
-# The working dir during testing
-PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-ETH2SPEC_MODULE_DIR = $(PY_SPEC_DIR)/eth2spec
-TEST_REPORT_DIR = $(PY_SPEC_DIR)/test-reports
-TEST_VECTOR_DIR = ../consensus-spec-tests/tests
-GENERATOR_DIR = ./tests/generators
-CONFIGS_DIR = ./configs
+all: help
+
+# A list of executable specifications.
+# These must pass a strict linter.
+ALL_EXECUTABLE_SPEC_NAMES = \
+	phase0    \
+	altair    \
+	bellatrix \
+	capella   \
+	deneb     \
+	electra   \
+	fulu      \
+	whisk     \
+	eip6800   \
+	eip7732
+
+# A list of fake targets.
+.PHONY: \
+	check_toc     \
+	clean         \
+	coverage      \
+	detect_errors \
+	eth2spec      \
+	gen_all       \
+	gen_list      \
+	help          \
+	kzg_setups    \
+	lint          \
+	pyspec        \
+	serve_docs    \
+	test
+
+###############################################################################
+# Help
+###############################################################################
+
+BOLD = $(shell tput bold)
+NORM = $(shell tput sgr0)
+
+# Print target descriptions.
+help:
+	@echo "make $(BOLD)check_toc$(NORM)     -- check table of contents"
+	@echo "make $(BOLD)clean$(NORM)         -- delete all untracked files"
+	@echo "make $(BOLD)coverage$(NORM)      -- run pyspec tests with coverage"
+	@echo "make $(BOLD)detect_errors$(NORM) -- detect generator errors"
+	@echo "make $(BOLD)eth2spec$(NORM)      -- force rebuild eth2spec package"
+	@echo "make $(BOLD)gen_<gen>$(NORM)     -- run a single generator"
+	@echo "make $(BOLD)gen_all$(NORM)       -- run all generators"
+	@echo "make $(BOLD)gen_list$(NORM)      -- list all generator targets"
+	@echo "make $(BOLD)kzg_setups$(NORM)    -- generate trusted setups"
+	@echo "make $(BOLD)lint$(NORM)          -- run the linters"
+	@echo "make $(BOLD)pyspec$(NORM)        -- generate python specifications"
+	@echo "make $(BOLD)serve_docs$(NORM)    -- start a local docs web server"
+	@echo "make $(BOLD)test$(NORM)          -- run pyspec tests"
+
+###############################################################################
+# Virtual Environment
+###############################################################################
+
+VENV = venv
+PYTHON_VENV = $(VENV)/bin/python3
+PIP_VENV = $(VENV)/bin/pip3
+CODESPELL_VENV = $(VENV)/bin/codespell
+
+# Make a virtual environment will all of the necessary dependencies.
+$(VENV): requirements_preinstallation.txt
+	@echo "Creating virtual environment"
+	@python3 -m venv $(VENV)
+	@$(PIP_VENV) install -r requirements_preinstallation.txt
+
+###############################################################################
+# Specification
+###############################################################################
+
+TEST_LIBS_DIR = $(CURDIR)/tests/core
+PYSPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+SITE_PACKAGES := $(wildcard $(VENV)/lib/python*/site-packages)
+ETH2SPEC := $(SITE_PACKAGES)/eth2spec
+
+# Install the eth2spec package.
+# The pipe indicates that venv is an order-only prerequisite.
+# When restoring venv cache, its timestamp is newer than eth2spec.
+$(ETH2SPEC): setup.py | $(VENV)
+	@$(PIP_VENV) install .[docs,lint,test,generator]
+
+# Force rebuild/install the eth2spec package.
+eth2spec:
+	$(MAKE) --always-make $(ETH2SPEC)
+
+# Create the pyspec for all phases.
+pyspec: $(VENV) setup.py
+	@echo "Building all pyspecs"
+	@$(PYTHON_VENV) setup.py pyspecdev
+
+###############################################################################
+# Testing
+###############################################################################
+
+TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
+
+# Run pyspec tests.
+# To run a specific test, append k=<test>, eg:
+#   make test k=test_verify_kzg_proof
+# To run tests for a specific fork, append fork=<fork>, eg:
+#   make test fork=deneb
+# To run tests for a specific preset, append preset=<preset>, eg:
+#   make test preset=mainnet
+# Or all at the same time, eg:
+#   make test preset=mainnet fork=deneb k=test_verify_kzg_proof
+# To run tests with a specific bls library, append bls=<bls>, eg:
+#   make test bls=arkworks
+test: MAYBE_TEST := $(if $(k),-k=$(k))
+test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
+test: PRESET := --preset=$(if $(preset),$(preset),minimal)
+test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
+test: $(ETH2SPEC) pyspec
+	@mkdir -p $(TEST_REPORT_DIR)
+	@$(PYTHON_VENV) -m pytest \
+		-n auto \
+		$(MAYBE_TEST) \
+		$(MAYBE_FORK) \
+		$(PRESET) \
+		$(BLS) \
+		--junitxml=$(TEST_REPORT_DIR)/test_results.xml \
+		$(PYSPEC_DIR)/eth2spec
+
+###############################################################################
+# Coverage
+###############################################################################
+
 TEST_PRESET_TYPE ?= minimal
-# Collect a list of generator names
-GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
-# Map this list of generator paths to "gen_{generator name}" entries
-GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
-GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
-# Documents
+COV_HTML_OUT=$(PYSPEC_DIR)/.htmlcov
+COV_INDEX_FILE=$(COV_HTML_OUT)/index.html
+COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
+
+# Run pytest with coverage tracking
+_test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
+_test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
+_test_with_coverage: $(ETH2SPEC) pyspec
+	@$(PYTHON_VENV) -m pytest \
+		-n auto \
+		$(MAYBE_TEST) \
+		$(MAYBE_FORK) \
+		--disable-bls \
+		$(COVERAGE_SCOPE) \
+		--cov-report="html:$(COV_HTML_OUT)" \
+		--cov-branch \
+		$(PYSPEC_DIR)/eth2spec
+
+# Run tests with coverage then open the coverage report.
+# See `make test` for a list of options.
+coverage: _test_with_coverage
+	@echo "Opening result: $(COV_INDEX_FILE)"
+	@((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
+
+###############################################################################
+# Documentation
+###############################################################################
+
 DOCS_DIR = ./docs
+FORK_CHOICE_DIR = ./fork_choice
+SPEC_DIR = ./specs
 SSZ_DIR = ./ssz
 SYNC_DIR = ./sync
-FORK_CHOICE_DIR = ./fork_choice
 
-# To check generator matching:
-#$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
+# Copy files to the docs directory.
+_copy_docs:
+	@cp -r $(SPEC_DIR) $(DOCS_DIR)
+	@cp -r $(SYNC_DIR) $(DOCS_DIR)
+	@cp -r $(SSZ_DIR) $(DOCS_DIR)
+	@cp -r $(FORK_CHOICE_DIR) $(DOCS_DIR)
+	@cp $(CURDIR)/README.md $(DOCS_DIR)/README.md
 
+# Start a local documentation server.
+serve_docs: _copy_docs
+	@mkdocs build
+	@mkdocs serve
+
+###############################################################################
+# Checks
+###############################################################################
+
+FLAKE8_CONFIG = $(CURDIR)/flake8.ini
+MYPY_CONFIG = $(CURDIR)/mypy.ini
+PYLINT_CONFIG = $(CURDIR)/pylint.ini
+
+PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), $(PYSPEC_DIR)/eth2spec/$S)
+MYPY_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), -p eth2spec.$S)
+TEST_GENERATORS_DIR = ./tests/generators
 MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/*/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra fulu whisk eip6800 eip7732
-# The parameters for commands. Use `foreach` to avoid listing specs again.
-COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
-PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), ./eth2spec/$S)
-MYPY_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), -p eth2spec.$S)
-
-COV_HTML_OUT=.htmlcov
-COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT)
-COV_INDEX_FILE=$(COV_HTML_OUT_DIR)/index.html
-
-CURRENT_DIR = ${CURDIR}
-GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt
-
-SCRIPTS_DIR = ${CURRENT_DIR}/scripts
-
-.PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \
-        check_toc detect_generator_incomplete detect_generator_error_log
-
-all: $(PY_SPEC_ALL_TARGETS)
-
-# deletes everything except the venvs
-partial_clean:
-	rm -rf $(TEST_VECTOR_DIR)
-	rm -rf $(GENERATOR_VENVS)
-	rm -rf .pytest_cache
-	rm -f .coverage
-	rm -rf $(PY_SPEC_DIR)/.pytest_cache
-	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
-	rm -rf $(COV_HTML_OUT_DIR)
-	rm -rf $(TEST_REPORT_DIR)
-	rm -rf eth2spec.egg-info dist build
-	rm -rf build;
-	@for spec_name in $(ALL_EXECUTABLE_SPEC_NAMES) ; do \
-		echo $$spec_name; \
-		rm -rf $(ETH2SPEC_MODULE_DIR)/$$spec_name; \
-	done
-
-clean: partial_clean
-	rm -rf venv
-	# legacy cleanup. The pyspec venv should be located at the repository root
-	rm -rf $(PY_SPEC_DIR)/venv
-	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/venv
-	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/venv
-
-# The pyspec is rebuilt to enforce the /specs being part of eth2specs source distribution. It could be forgotten otherwise.
-dist_build: pyspec
-	python3 setup.py sdist bdist_wheel
-
-dist_check:
-	python3 -m twine check dist/*
-
-dist_upload:
-	python3 -m twine upload dist/*
-
-build_wheel: install_test pyspec
-	. venv/bin/activate && \
-	python3 -m build --no-isolation --outdir ./dist ./
-
-# "make generate_tests" to run all generators
-generate_tests: $(GENERATOR_TARGETS)
-
-# "make pyspec" to create the pyspec for all phases.
-pyspec:
-	@python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
-
-# check the setup tool requirements
-preinstallation:
-	python3 -m venv venv && \
-	. venv/bin/activate && \
-	python3 -m pip install -r requirements_preinstallation.txt
-
-install_test: preinstallation
-	. venv/bin/activate && \
-	python3 -m pip install -e .[lint,test]
-
-# Testing against `minimal` or `mainnet` config by default
-test: pyspec
-	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n auto --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
-
-# Testing against `minimal` or `mainnet` config by default
-find_test: pyspec
-	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
-
-citest: pyspec
-	mkdir -p $(TEST_REPORT_DIR);
-ifdef fork
-	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
-else
-	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
-endif
-
-
-open_cov:
-	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
-
 # Check all files and error if any ToC were modified.
 check_toc: $(MARKDOWN_FILES:=.toc)
 	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
@@ -145,86 +209,67 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 		echo "\033[1;34m See $*.tmp\033[0m"; \
 	fi
 
-codespell:
-	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
-
-lint: pyspec
-	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	flake8  --config $(CURRENT_DIR)/flake8.ini ./eth2spec \
-	&& python -m pylint --rcfile $(CURRENT_DIR)/pylint.ini $(PYLINT_SCOPE) \
-	&& python -m mypy --config-file $(CURRENT_DIR)/mypy.ini $(MYPY_SCOPE)
-
-lint_generators: pyspec
-	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \
-	flake8 --config $(CURRENT_DIR)/flake8.ini
-
-# If set to true, it will not run generator tests.
-modcheck ?= false
-
-# Runs a generator, identified by param 1
-define run_generator
-	# Started!
-	# Create output directory
-	# Navigate to the generator
-	# Create a virtual environment, if it does not exist already
-	# Activate the venv, this is where dependencies are installed for the generator
-	# Install all the necessary requirements
-	# Run the generator. The generator is assumed to have an "main.py" file.
-	# We output to the tests dir (generator program should accept a "-o <filepath>" argument.
-	# `-l minimal general` can be added to the generator call to filter to smaller configs, when testing.
-	echo "generator $(1) started"; \
-	mkdir -p $(TEST_VECTOR_DIR); \
-	cd $(GENERATOR_DIR)/$(1); \
-	if ! test -d venv; then python3 -m venv venv; fi; \
-	. venv/bin/activate; \
-	pip3 install ../../../dist/eth2spec-*.whl; \
-	pip3 install 'eth2spec[generator]'; \
-	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(modcheck)),--modcheck); \
-	echo "generator $(1) finished"
-endef
-
-# The tests dir itself is simply built by creating the directory (recursively creating deeper directories if necessary)
-$(TEST_VECTOR_DIR):
-	$(info creating test output directory, for generators: ${GENERATOR_TARGETS})
-	mkdir -p $@
-$(TEST_VECTOR_DIR)/:
-	$(info ignoring duplicate tests dir)
-
-gen_kzg_setups:
-	cd $(SCRIPTS_DIR); \
-	if ! test -d venv; then python3 -m venv venv; fi; \
-	. venv/bin/activate; \
-	pip3 install -r requirements.txt; \
-	python3 ./gen_kzg_trusted_setups.py --secret=1337 --g1-length=4096 --g2-length=65 --output-dir ${CURRENT_DIR}/presets/minimal/trusted_setups; \
-	python3 ./gen_kzg_trusted_setups.py --secret=1337 --g1-length=4096 --g2-length=65 --output-dir ${CURRENT_DIR}/presets/mainnet/trusted_setups
-
-# For any generator, build it using the run_generator function.
-# (creation of output dir is a dependency)
-gen_%: build_wheel $(TEST_VECTOR_DIR)
-	$(call run_generator,$*)
-
-detect_generator_incomplete: $(TEST_VECTOR_DIR)
-	find $(TEST_VECTOR_DIR) -name "INCOMPLETE"
-
-detect_generator_error_log: $(TEST_VECTOR_DIR)
-	[ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist"
-
-
-# For docs reader
-install_docs:
-	python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[docs];
-
-copy_docs:
-	cp -r $(SPEC_DIR) $(DOCS_DIR);
-	cp -r $(SYNC_DIR) $(DOCS_DIR);
-	cp -r $(SSZ_DIR) $(DOCS_DIR);
-	cp -r $(FORK_CHOICE_DIR) $(DOCS_DIR);
-	cp $(CURRENT_DIR)/README.md $(DOCS_DIR)/README.md
-
-build_docs: copy_docs
-	. venv/bin/activate;
-	mkdocs build
-
-serve_docs:
-	. venv/bin/activate;
-	mkdocs serve
+# Check for mistakes.
+lint: $(ETH2SPEC) pyspec check_toc
+	@$(CODESPELL_VENV) . --skip "./.git,$(VENV),$(PYSPEC_DIR)/.mypy_cache" -I .codespell-whitelist
+	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(PYSPEC_DIR)/eth2spec
+	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(TEST_GENERATORS_DIR)
+	@$(PYTHON_VENV) -m pylint --rcfile $(PYLINT_CONFIG) $(PYLINT_SCOPE)
+	@$(PYTHON_VENV) -m mypy --config-file $(MYPY_CONFIG) $(MYPY_SCOPE)
+
+###############################################################################
+# Generators
+###############################################################################
+
+TEST_VECTOR_DIR = $(CURDIR)/../consensus-spec-tests/tests
+GENERATOR_DIR = $(CURDIR)/tests/generators
+SCRIPTS_DIR = $(CURDIR)/scripts
+GENERATOR_ERROR_LOG_FILE = $(TEST_VECTOR_DIR)/testgen_error_log.txt
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
+GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
+
+# List available generators.
+gen_list:
+	@for target in $(shell echo $(GENERATOR_TARGETS) | tr ' ' '\n' | sort -n); do \
+		echo $$target; \
+	done
+
+# Run one generator.
+# To check modules for a generator, append modcheck=true, eg:
+#   make gen_genesis modcheck=true
+gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
+gen_%: $(ETH2SPEC) pyspec
+	@mkdir -p $(TEST_VECTOR_DIR)
+	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
+		--output $(TEST_VECTOR_DIR) \
+		$(MAYBE_MODCHECK)
+
+# Run all generators then check for errors.
+gen_all: $(GENERATOR_TARGETS) detect_errors
+
+# Detect errors in generators.
+detect_errors: $(TEST_VECTOR_DIR)
+	@find $(TEST_VECTOR_DIR) -name "INCOMPLETE"
+	@if [ -f $(GENERATOR_ERROR_LOG_FILE) ]; then \
+		echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists"; \
+	else \
+		echo "[PASSED] error log file does not exist"; \
+	fi
+
+# Generate KZG trusted setups for testing.
+kzg_setups: $(ETH2SPEC)
+	@for preset in minimal mainnet; do \
+		$(PYTHON_VENV) $(SCRIPTS_DIR)/gen_kzg_trusted_setups.py \
+			--secret=1337 \
+			--g1-length=4096 \
+			--g2-length=65 \
+			--output-dir $(CURDIR)/presets/$$preset/trusted_setups; \
+	done
+
+###############################################################################
+# Cleaning
+###############################################################################
+
+# Delete all untracked files.
+clean:
+	@git clean -fdx
\ No newline at end of file
diff --git a/README.md b/README.md
index 21ca42dddb..e58927ac9e 100644
--- a/README.md
+++ b/README.md
@@ -88,8 +88,7 @@ The consensus-specs repo can be used by running the tests locally or inside a do
 To run the tests locally:
 - Clone the repository with `git clone https://github.com/ethereum/consensus-specs.git`
 - Switch to the directory `cd consensus-specs`
-- Install the dependencies with: `make install_test && make preinstallation && make pyspec`
-- Run the tests with `make citest`
+- Run the tests with `make test`
 
 To run the tests inside a docker container:
 - Switch to the directory with `cd scripts`
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 8ec384499d..d4b2d3567a 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -16,7 +16,4 @@ RUN apt update && ${INSTALL_CMD} && ${PIP_UPGRADE_CMD} && rm -rf /var/lib/apt/li
 COPY . .
 
 # Inline installation commands
-RUN make install_test && \
-    make preinstallation && \
-    make pyspec
-
+RUN make pyspec
diff --git a/docker/README.md b/docker/README.md
index 44dc2b95e5..4824fc283a 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -6,7 +6,7 @@ This dockerfile sets up the dependencies required to run consensus-spec tests. T
 
 Handy commands:
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
-- `docker run $IMAGE_NAME make citest` will run the make citest command inside the docker container
+- `docker run $IMAGE_NAME make test` will run the make test command inside the docker container
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
diff --git a/scripts/build_run_docker_tests.sh b/scripts/build_run_docker_tests.sh
index 1716a4774c..24d70b145d 100755
--- a/scripts/build_run_docker_tests.sh
+++ b/scripts/build_run_docker_tests.sh
@@ -82,16 +82,16 @@ else
     echo "Image $IMAGE_NAME already exists. Skipping build..."
 fi
 
-# Equivalent to `make citest with the subsequent flags`
+# Equivalent to `make test with the subsequent flags`
 if [ "$FORK_TO_TEST" == "all" ]; then
   for fork in "${ALL_EXECUTABLE_SPECS[@]}"; do
     docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE
+      make test fork=$fork preset=$TEST_PRESET_TYPE
       copy_test_results $fork
   done
 else
   docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE
+      make test fork=$FORK_TO_TEST preset=$TEST_PRESET_TYPE
   copy_test_results $FORK_TO_TEST
 fi
 
diff --git a/setup.py b/setup.py
index 83ae7accb4..0bc90ae787 100644
--- a/setup.py
+++ b/setup.py
@@ -561,7 +561,7 @@ def run(self):
     python_requires=">=3.9, <4",
     extras_require={
         "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
-        "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1"],
+        "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1", "codespell<3.0.0,>=2.0.0"],
         "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"],
         "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3",  "mkdocs-awesome-pages-plugin==2.8.0"]
     },
diff --git a/tests/README.md b/tests/README.md
index 1ffa17239a..798627577d 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -17,48 +17,29 @@ Use an OS that has Python 3.8 or above. For example, Debian 11 (bullseye)
    ```
 3. Create the specifications and tests:
    ```sh
-   make install_test
-   make pyspec
+   make
    ```
 
 To read more about creating the environment, [see here](core/pyspec/README.md).
 
 ### Running your first test
 
+Use `make` to run the `test_empty_block_transition` tests against the Altair fork like so:
 
-1. Enter the virtual Python environment:
-   ```sh
-   cd ~/consensus-specs
-   . venv/bin/activate
-   ```
-2. Run a sanity check test against Altair fork:
-   ```sh
-   cd tests/core/pyspec
-   python -m pytest -k test_empty_block_transition --fork altair eth2spec
-   ```
-3. The output should be similar to:
-   ```
-   ============================= test session starts ==============================
-   platform linux -- Python 3.9.2, pytest-6.2.5, py-1.10.0, pluggy-1.0.0
-   rootdir: /home/qbzzt1/consensus-specs
-   plugins: cov-2.12.1, forked-1.3.0, xdist-2.3.0
-   collected 629 items / 626 deselected / 3 selected
-
-   eth2spec/test/bellatrix/sanity/test_blocks.py .                              [ 33%]
-   eth2spec/test/phase0/sanity/test_blocks.py ..                            [100%]
-
-   =============================== warnings summary ===============================
-   ../../../venv/lib/python3.9/site-packages/cytoolz/compatibility.py:2
-     /home/qbzzt1/consensus-specs/venv/lib/python3.9/site-packages/cytoolz/compatibility.py:2:
-   DeprecationWarning: The toolz.compatibility module is no longer needed in Python 3 and has
-   been deprecated. Please import these utilities directly from the standard library. This
-   module will be removed in a future release.
-       warnings.warn("The toolz.compatibility module is no longer "
-
-   -- Docs: https://docs.pytest.org/en/stable/warnings.html
-   ================ 3 passed, 626 deselected, 1 warning in 16.81s =================
-   ```
-
+```
+$ make test k=test_empty_block_transition fork=altair
+Building all pyspecs
+...
+================================= test session starts ==================================
+platform darwin -- Python 3.10.3, pytest-8.3.3, pluggy-1.5.0
+rootdir: /Users/jtraglia/Projects/jtraglia/consensus-specs
+plugins: cov-5.0.0, xdist-3.6.1
+20 workers [3 items]
+s..                                                                              [100%]
+=================================== warnings summary ===================================
+-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+====================== 2 passed, 1 skipped, 42 warnings in 7.97s =======================
+```
 
 ## The "Hello, World" of Consensus Spec Tests
 
@@ -451,10 +432,7 @@ Add this function to the file `consensus-specs/tests/core/pyspec/eth2spec/test/p
 and run the test against Altair fork:
 
 ```sh
-cd ~/consensus-specs
-. venv/bin/activate
-cd tests/core/pyspec
-python -m pytest -k almost_after --fork altair eth2spec
+make test k=almost_after fork=altair
 ```
 
 You should see it ran successfully (although you might get a warning, you can ignore it)
diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md
index baa1322771..2fdd107fb1 100644
--- a/tests/core/pyspec/README.md
+++ b/tests/core/pyspec/README.md
@@ -7,29 +7,6 @@ With this executable spec,
  test-generators can easily create test-vectors for client implementations,
  and the spec itself can be verified to be consistent and coherent through sanity tests implemented with pytest.
 
-## Dev Install
-
-First, create a `venv` and install the developer dependencies (`test` and `lint` extras):
-
-```shell
-make install_test
-```
-
-All the dynamic parts of the spec are built with:
-
-```shell
-(venv) python setup.py pyspecdev
-```
-
-Unlike the regular install, this outputs spec files to their intended source location,
-to enable debuggers to navigate between packages and generated code, without fragile directory linking.
-
-By default, when installing the `eth2spec` as package in non-develop mode,
-the distutils implementation of the `setup` runs `build`, which is extended to run the same `pyspec` work,
-but outputs into the standard `./build/lib` output.
-This enables the `consensus-specs` repository to be installed like any other python package.
-
-
 ## Py-tests
 
 These tests are not intended for client-consumption.
@@ -38,61 +15,41 @@ However, most of the tests can be run in generator-mode, to output test vectors
 
 ### How to run tests
 
-#### Automated
-
-Run `make test` from the root of the specs repository (after running `make install_test` if have not before).
-
-Note that the `make` commands run through the build steps: it runs the `build` output, not the local package source files.
-
-#### Manual
-
-See `Dev install` for test pre-requisites.
-
-Tests are built for `pytest`.
-
-Caveats:
-- Working directory must be `./tests/core/pyspec`. The work-directory is important to locate eth2 configuration files.
-- Run `pytest` as module. It avoids environment differences, and the behavior is different too:
-  `pytest` as module adds the current directory to the `sys.path`
+To run all tests:
+```shell
+make test
+```
 
-Full test usage, with explicit configuration for illustration of options usage:
+To run all tests under the minimal preset:
 ```shell
-(venv) python -m pytest --preset=minimal eth2spec
+make test preset=minimal
 ```
 
-Or, to run a specific test file, specify the full path:
+Or, to run a specific test function specify `k=<test-name>`:
 ```shell
-(venv) python -m pytest --preset=minimal ./eth2spec/test/phase0/block_processing/test_process_attestation.py
+make test k=test_verify_kzg_proof
 ```
 
-Or, to run a specific test function (specify the `eth2spec` module, or the script path if the keyword is ambiguous):
+Or, to run a specific test function under a single fork specify `k=<test-name>`:
 ```shell
-(venv) python -m pytest --preset=minimal -k test_success_multi_proposer_index_iterations eth2spec
+make test fork=phase0
 ```
 
-Options:
-- `--preset`, to change the preset (compile-time configurables). Defaults to `minimal`, can be set to `mainnet`.
-  Use `@spec_configured_state_test({config here...}` to override runtime configurables on a per-test basis.
-- `--disable-bls`, to disable BLS (only for tests that can run without)
-- `--bls-type`, `milagro` or `py_ecc` (default)
+Note: these options can be used together, like:
 
-### How to view code coverage report
-
-Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report.
+```shell
+make test preset=minimal k=test_verify_kzg_proof fork=deneb
+```
 
-### Advanced
+### How to view code coverage report
 
-Building spec files from any markdown sources, to a custom location:
-```bash
-(venv) python setup.py pyspec --spec-fork=phase0 --md-doc-paths="specs/phase0/beacon-chain.md specs/phase0/fork-choice.md" --out-dir=my_spec_dir
-```
+Run `make coverage` to run all tests and open the html code coverage report.
 
 ## Contributing
 
 Contributions are welcome, but consider implementing your idea as part of the spec itself first.
 The pyspec is not a replacement.
 
-
 ## License
 
 Same as the spec itself; see [LICENSE](../../../LICENSE) file in the specs repository root.
diff --git a/tests/generators/README.md b/tests/generators/README.md
index 993c1e3472..270d107ea5 100644
--- a/tests/generators/README.md
+++ b/tests/generators/README.md
@@ -39,7 +39,7 @@ Prerequisites:
 This removes the existing virtual environments (`/tests/generators/<generator>/venv`) and generated tests (`../consensus-spec-tests/tests`).
 
 ```bash
-make clean
+make clean && rm -rf ../consensus-spec-tests/tests
 ```
 
 ### Running all test generators
@@ -47,7 +47,7 @@ make clean
 This runs all of the generators.
 
 ```bash
-make -j 4 generate_tests
+make -j 4 gen_all
 ```
 
 The `-j N` flag makes the generators run in parallel, with `N` being the amount of cores.
@@ -63,39 +63,12 @@ make gen_ssz_static
 
 ## Developing a generator
 
-Simply open up the generator (not all at once) of choice in your favorite IDE/editor and run:
-
-```bash
-# From the root of the generator directory:
-# Create a virtual environment (any venv/.venv/.venvs is git-ignored)
-python3 -m venv venv
-# Activate the venv, this is where dependencies are installed for the generator
-. venv/bin/activate
-```
-
-Now that you have a virtual environment, write your generator.
-It's recommended to extend the base-generator.
-
-Create a `requirements.txt` in the root of your generator directory:
-```
-pytest>=4.4
-../../../[generator]
-```
-
 The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests.
 Applying configurations to the spec is simple and enables you to create test suites with different contexts.
 
 *Note*: Make sure to run `make pyspec` from the root of the specs repository in order to build the pyspec requirement.
 
-Install all the necessary requirements (re-run when you add more):
-```bash
-pip3 install -r requirements.txt
-```
-
-Note that you may need `PYTHONPATH` to include the pyspec directory, as with running normal tests,
- to run test generators manually. The makefile handles this for you already.
-
-And write your initial test generator, extending the base generator:
+Write your initial test generator, extending the base generator:
 
 Write a `main.py` file. The shuffling test generator is a good minimal starting point: