Merge branch 'dev' into eip6110-tests

.github/workflows/docs.yml

@@ -0,0 +1,24 @@
+
+name: Publish docs
+on:
+  push:
+    branches:
+      - master 
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build docs
+        run: make copy_docs
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - uses: actions/cache@v2
+        with:
+          key: ${{ github.ref }}
+          path: .cache
+      - run: pip install -e .[docs]
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -35,3 +35,11 @@ tests/core/pyspec/eth2spec/test_results.xml
 
 # TOC tool outputs temporary files
 *.tmp
+
+# docs reader build
+docs/specs
+docs/sync
+docs/ssz
+docs/fork_choice
+docs/README.md
+site

Makefile

@@ -19,6 +19,11 @@ GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 # Map this list of generator paths to "gen_{generator name}" entries
 GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
 GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
+# Documents
+DOCS_DIR = ./docs
+SSZ_DIR = ./ssz
+SYNC_DIR = ./sync
+FORK_CHOICE_DIR = ./fork_choice
 
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
@@ -214,3 +219,23 @@ detect_generator_incomplete: $(TEST_VECTOR_DIR)
 
 detect_generator_error_log: $(TEST_VECTOR_DIR)
 	[ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist"
+
+
+# For docs reader
+install_docs:
+	python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[docs];
+
+copy_docs:
+	cp -r $(SPEC_DIR) $(DOCS_DIR);
+	cp -r $(SYNC_DIR) $(DOCS_DIR);
+	cp -r $(SSZ_DIR) $(DOCS_DIR);
+	cp -r $(FORK_CHOICE_DIR) $(DOCS_DIR);
+	cp $(CURRENT_DIR)/README.md $(DOCS_DIR)/README.md
+
+build_docs: copy_docs
+	. venv/bin/activate;
+	mkdocs build
+
+serve_docs:
+	. venv/bin/activate; 
+	mkdocs serve

README.md

@@ -1,6 +1,6 @@
 # Ethereum Proof-of-Stake Consensus Specifications
 
-[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA)
 
 To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm).
 
@@ -65,6 +65,10 @@ Documentation on the different components used during spec writing can be found
 * [YAML Test Generators](tests/generators/README.md)
 * [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md)
 
+## Online viewer of the latest release (latest `master` branch)
+
+[Ethereum Consensus Specs](https://ethereum.github.io/consensus-specs/)
+
 ## Consensus spec tests
 
 Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).

docs/.pages

@@ -0,0 +1,5 @@
+nav:
+    - Home:
+        - README.md
+    - specs
+    - ...

docs/docs/new-feature.md

@@ -0,0 +1,163 @@
+# How to add a new feature proposal in consensus-specs
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+## Table of Contents
+
+- [A. Make it executable for linter checks](#a-make-it-executable-for-linter-checks)
+  - [1. Create a folder under `./specs/_features`](#1-create-a-folder-under-specs_features)
+  - [2. Choose the "previous fork" to extend: usually, use the scheduled or the latest mainnet fork version.](#2-choose-the-previous-fork-to-extend-usually-use-the-scheduled-or-the-latest-mainnet-fork-version)
+  - [3. Write down your proposed `beacon-chain.md` change](#3-write-down-your-proposed-beacon-chainmd-change)
+  - [4. Add `fork.md`](#4-add-forkmd)
+  - [5. Make it executable](#5-make-it-executable)
+- [B: Make it executable for pytest and test generator](#b-make-it-executable-for-pytest-and-test-generator)
+  - [1. Add `light-client/*` docs if you updated the content of `BeaconBlock`](#1-add-light-client-docs-if-you-updated-the-content-of-beaconblock)
+  - [2. Add the mainnet and minimal presets and update the configs](#2-add-the-mainnet-and-minimal-presets-and-update-the-configs)
+  - [3. Update `context.py`](#3-update-contextpy)
+  - [4. Update `constants.py`](#4-update-constantspy)
+  - [5. Update `genesis.py`:](#5-update-genesispy)
+  - [6. To add fork transition tests, update fork_transition.py](#6-to-add-fork-transition-tests-update-fork_transitionpy)
+  - [7. Update CI configurations](#7-update-ci-configurations)
+- [Others](#others)
+  - [Bonus](#bonus)
+  - [Need help?](#need-help)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+
+## A. Make it executable for linter checks
+
+### 1. Create a folder under `./specs/_features`
+
+For example, if it's an `EIP-9999` CL spec, you can create a `./specs/_features/eip9999` folder.
+
+### 2. Choose the "previous fork" to extend: usually, use the scheduled or the latest mainnet fork version.
+
+For example, if the latest fork is Capella, use `./specs/capella` content as your "previous fork".
+
+### 3. Write down your proposed `beacon-chain.md` change
+- You can either use [Beacon Chain Spec Template](./templates/beacon-chain-template.md), or make a copy of the latest fork content and then edit it.
+- Tips:
+    - We use [`doctoc`](https://www.npmjs.com/package/doctoc) tool to generate the table of content.
+        ```
+        cd consensus-specs
+        doctoc specs
+        ```
+    - The differences between "Constants", "Configurations", and "Presets":
+        - Constants: The constant that should never be changed.
+        - Configurations: The settings that we may change for different networks.
+        - Presets: The settings that we may change for testing.
+    - Readability and simplicity are more important than efficiency and optimization.
+        - Use simple Python rather than the fancy Python dark magic.
+
+### 4. Add `fork.md`
+You can refer to the previous fork's `fork.md` file.
+### 5. Make it executable
+- Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name.
+- Update [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py):
+    - Add a new `SpecBuilder` with the new feature name constant. e.g., `EIP9999SpecBuilder`
+    - Add the new `SpecBuilder` to `spec_builders` list.
+    - Add the path of the new markdown files in `finalize_options` function.
+
+## B: Make it executable for pytest and test generator
+
+### 1. Add `light-client/*` docs if you updated the content of `BeaconBlock`
+- You can refer to the previous fork's `light-client/*` file.
+- Add the path of the new markdown files in `setup.py`'s `finalize_options` function.
+
+### 2. Add the mainnet and minimal presets and update the configs
+- Add presets: `presets/mainnet/<new-feature-name>.yaml` and `presets/minimal/<new-feature-name>.yaml`
+- Update configs: `configs/mainnet.yaml` and `configs/minimal.yaml`
+
+### 3. Update [`context.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/context.py)
+- Update `spec_targets` by adding `<NEW_FEATURE>`
+
+```python
+from eth2spec.eip9999 import mainnet as spec_eip9999_mainnet, minimal as spec_eip9999_minimal
+
+...
+
+spec_targets: Dict[PresetBaseName, Dict[SpecForkName, Spec]] = {
+    MINIMAL: {
+        ...
+        EIP9999: spec_eip9999_minimal,
+    },
+    MAINNET: {
+        ...
+        EIP9999: spec_eip9999_mainnet
+    },
+}
+```
+
+### 4. Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py)
+- Add `<NEW_FEATURE>` to `ALL_PHASES` and `TESTGEN_FORKS`
+
+### 5. Update [`genesis.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/genesis.py):
+
+We use `create_genesis_state` to create the default `state` in tests.
+
+- Update `create_genesis_state` by adding `fork_version` setting:
+
+```python
+def create_genesis_state(spec, validator_balances, activation_threshold):
+    ...
+    if spec.fork == ALTAIR:
+        current_version = spec.config.ALTAIR_FORK_VERSION
+    ...
+    elif spec.fork == EIP9999:
+        # Add the previous fork version of given fork
+        previous_version = spec.config.<PREVIOUS_FORK_VERSION>
+        current_version = spec.config.EIP9999_FORK_VERSION
+```
+
+- If the given feature changes `BeaconState` fields, you have to set the initial values by adding:
+
+```python
+def create_genesis_state(spec, validator_balances, activation_threshold):
+    ...
+    if is_post_eip9999(spec):
+        state.<NEW_FIELD> = <value>
+
+    return state
+```
+
+- If the given feature changes `ExecutionPayload` fields, you have to set the initial values by updating `get_sample_genesis_execution_payload_header` helper.
+
+### 6. To add fork transition tests, update [fork_transition.py](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py)
+
+```python
+def do_fork(state, spec, post_spec, fork_epoch, with_block=True, sync_aggregate=None, operation_dict=None):
+    ...
+
+    if post_spec.fork == ALTAIR:
+        state = post_spec.upgrade_to_altair(state)
+    ...
+    elif post_spec.fork == EIP9999:
+        state = post_spec.upgrade_to_eip9999(state)
+
+    ...
+
+    if post_spec.fork == ALTAIR:
+        assert state.fork.previous_version == post_spec.config.GENESIS_FORK_VERSION
+        assert state.fork.current_version == post_spec.config.ALTAIR_FORK_VERSION
+    ...
+    elif post_spec.fork == EIP9999:
+        assert state.fork.previous_version == post_spec.config.<PREVIOUS_FORK_VERSION>
+        assert state.fork.current_version == post_spec.config.EIP9999_FORK_VERSION
+
+    ...
+```
+
+### 7. Update CI configurations
+- Update [GitHub Actions config](https://github.com/ethereum/consensus-specs/blob/dev/.github/workflows/run-tests.yml)
+    - Update `pyspec-tests.strategy.matrix.version` list by adding new feature to it
+- Update [CircleCI config](https://github.com/ethereum/consensus-specs/blob/dev/.circleci/config.yml)
+    - Add new job to the `workflows.test_spec.jobs`
+
+## Others
+
+### Bonus
+- Add `validator.md` if honest validator behavior changes with the new feature.
+
+### Need help?
+You can tag spec elves for cleaning up your PR. 🧚

docs/docs/templates/beacon-chain-template.md

@@ -0,0 +1,84 @@
+# `beacon-chain.md` Template
+
+# <FORK_NAME> -- The Beacon Chain
+
+## Table of contents
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+
+
+## Introduction
+
+## Notation
+
+## Custom types
+
+## Constants
+
+### [CATEGORY OF CONSTANTS]
+
+| Name | Value |
+| - | - |
+| `<CONSTANT_NAME>` | `<VALUE>`` |
+
+## Preset
+
+
+### [CATEGORY OF PRESETS]
+
+| Name | Value |
+| - | - |
+| `<PRESET_FIELD_NAME>` | `<VALUE>` |
+
+## Configuration
+
+### [CATEGORY OF CONFIGURATIONS]
+
+| Name | Value |
+| - | - |
+| `<CONFIGURATION_FIELD_NAME>` | `<VALUE>` |
+
+## Containers
+
+### [CATEGORY OF CONTAINERS]
+
+#### `CONTAINER_NAME`
+
+```python
+class CONTAINER_NAME(Container):
+    FILED_NAME: SSZ_TYPE
+```
+
+## Helper functions
+
+### [CATEGORY OF HELPERS]
+
+```python
+<PYTHON HELPER FUNCTION>
+```
+
+### Epoch processing
+
+
+### Block processing
+
+
+
+
+## Testing
+
+*Note*: The function `initialize_beacon_state_from_eth1` is modified for pure <FORK_NAME> testing only.
+
+```python
+def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32,
+                                      eth1_timestamp: uint64,
+                                      deposits: Sequence[Deposit],
+                                      execution_payload_header: ExecutionPayloadHeader=ExecutionPayloadHeader()
+                                      ) -> BeaconState:
+    ...
+```

docs/light-client/.pages

@@ -0,0 +1,5 @@
+nav:
+    - 'Index': index.md
+    - 'Altair': specs/altair/light-client/sync-protocol
+    - 'Capella': specs/capella/light-client/sync-protocol
+    - 'Deneb': specs/deneb/light-client/sync-protocol

docs/light-client/index.md

@@ -0,0 +1 @@
+# Light client specifications

docs/stylesheets/extra.css

@@ -0,0 +1,34 @@
+/* Reference: https://zenn.dev/mebiusbox/articles/81d977a72cee01 */
+
+[data-md-color-scheme=default] {
+  --md-default-fg-color--light: #222 !important;
+}
+[data-md-color-scheme=slate] {
+  --md-default-fg-color--light: #fefefe !important;
+  --md-typeset-a-color: #fc0 !important;
+}
+
+.md-typeset pre {
+    color: #f8f8f2;
+}
+.md-typeset .highlighttable {
+    margin-left:-20px;
+    margin-right: -20px;
+    border-radius: 0;
+}
+.md-typeset .highlighttable > * {
+    --md-code-bg-color: #222 !important;
+    --md-code-fg-color: #fefefe !important;
+}
+.md-typeset .highlighttable .linenos .linenodiv pre span {
+    background-color: #222 !important;
+    color: #fefefe !important;
+}
+.md-typeset .highlighttable .md-clipboard:before,
+.md-typeset .highlighttable .md-clipboard:after {
+    color: rgba(240,240,240,.8);
+}
+.md-typeset .highlighttable .md-clipboard:hover:before,
+.md-typeset .highlighttable .md-clipboard:hover:after {
+    color: rgba(102,217,224,1);
+}

fork_choice/.pages

@@ -0,0 +1,7 @@
+nav:
+    - ...
+    - Fork Choice -- Core:
+        - phase0: specs/phase0/fork-choice
+        - bellatrix: specs/bellatrix/fork-choice
+        - capella: specs/capella/fork-choice
+        - deneb: specs/deneb/fork-choice