Merge branch 'dev' into hwwhww/new_int_to_bytes_dev

ethereum · Jul 15, 2020 · 8e82ca5 · 8e82ca5
2 parents 4b239e9 + 6a7a47d
commit 8e82ca5
Show file tree

Hide file tree

Showing 25 changed files with 1,287 additions and 389 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,39 +35,39 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v19-pyspec
+          venv_name: v22-pyspec
           reqs_checksum: cache-{{ checksum "setup.py" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v19-pyspec
+          venv_name: v22-pyspec
           reqs_checksum: cache-{{ checksum "setup.py" }}
           venv_path: ./venv
   restore_deposit_contract_compiler_cached_venv:
     description: "Restore the venv from cache for the deposit contract compiler"
     steps:
       - restore_cached_venv:
-          venv_name: v18-deposit-contract-compiler
+          venv_name: v23-deposit-contract-compiler
           reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
   save_deposit_contract_compiler_cached_venv:
     description: "Save the venv to cache for later use of the deposit contract compiler"
     steps:
       - save_cached_venv:
-          venv_name: v18-deposit-contract-compiler
+          venv_name: v23-deposit-contract-compiler
           reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
           venv_path: ./deposit_contract/compiler/venv
   restore_deposit_contract_tester_cached_venv:
     description: "Restore the venv from cache for the deposit contract tester"
     steps:
       - restore_cached_venv:
-          venv_name: v19-deposit-contract-tester
+          venv_name: v22-deposit-contract-tester
           reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
   save_deposit_contract_tester_cached_venv:
     description: "Save the venv to cache for later use of the deposit contract tester"
     steps:
       - save_cached_venv:
-          venv_name: v19-deposit-contract-tester
+          venv_name: v22-deposit-contract-tester
           reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
           venv_path: ./deposit_contract/tester/venv
 jobs:

diff --git a/setup.py b/setup.py
@@ -94,7 +94,7 @@ def get_spec(file_name: str) -> SpecObject:
 
 from lru import LRU
 
-from eth2spec.utils.ssz.ssz_impl import hash_tree_root, uint_to_bytes
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
 from eth2spec.utils.ssz.ssz_typing import (
     View, boolean, Container, List, Vector, uint8, uint32, uint64,
     Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
@@ -118,7 +118,7 @@ def get_spec(file_name: str) -> SpecObject:
 
 from lru import LRU
 
-from eth2spec.utils.ssz.ssz_impl import hash_tree_root, uint_to_bytes
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
 from eth2spec.utils.ssz.ssz_typing import (
     View, boolean, Container, List, Vector, uint8, uint32, uint64, bit,
     ByteList, ByteVector, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
@@ -216,11 +216,21 @@ def wrapper(*args, **kw):  # type: ignore
 
 _get_attesting_indices = get_attesting_indices
 get_attesting_indices = cache_this(
-    lambda state, data, bits: (state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root()),
+    lambda state, data, bits: (
+        state.randao_mixes.hash_tree_root(),
+        state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root()
+    ),
     _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)'''
 
 
 PHASE1_SUNDRY_FUNCTIONS = '''
+
+def get_block_data_merkle_root(data: ByteList) -> Root:
+    # To get the Merkle root of the block data, we need the Merkle root without the length mix-in
+    # The below implements this in the Remerkleable framework
+    return Root(data.get_backing().get_left().merkle_root())
+
+
 _get_start_shard = get_start_shard
 get_start_shard = cache_this(
     lambda state, slot: (state.validators.hash_tree_root(), slot),
@@ -519,7 +529,7 @@ def run(self):
         "py_ecc==4.0.0",
         "milagro_bls_binding==1.3.0",
         "dataclasses==0.6",
-        "remerkleable==0.1.16",
+        "remerkleable==0.1.17",
         "ruamel.yaml==0.16.5",
         "lru-dict==1.1.6"
     ]

diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md
@@ -170,9 +170,9 @@ The following values are (non-configurable) constants used throughout the specif
 | `GENESIS_SLOT` | `Slot(0)` |
 | `GENESIS_EPOCH` | `Epoch(0)` |
 | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` |
-| `BASE_REWARDS_PER_EPOCH` | `4` |
-| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) |
-| `JUSTIFICATION_BITS_LENGTH` | `4` |
+| `BASE_REWARDS_PER_EPOCH` | `uint64(4)` |
+| `DEPOSIT_CONTRACT_TREE_DEPTH` | `uint64(2**5)` (= 32) |
+| `JUSTIFICATION_BITS_LENGTH` | `uint64(4)` |
 | `ENDIANNESS` | `'little'` |
 
 ## Configuration
@@ -183,18 +183,18 @@ The following values are (non-configurable) constants used throughout the specif
 
 | Name | Value |
 | - | - |
-| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) |
-| `MAX_COMMITTEES_PER_SLOT` | `2**6` (= 64) |
-| `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) |
-| `MAX_VALIDATORS_PER_COMMITTEE` | `2**11` (= 2,048) |
-| `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) |
-| `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) |
-| `SHUFFLE_ROUND_COUNT` | `90` |
-| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**14` (= 16,384) |
-| `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) |
-| `HYSTERESIS_QUOTIENT` | `4` |
-| `HYSTERESIS_DOWNWARD_MULTIPLIER` | `1` |
-| `HYSTERESIS_UPWARD_MULTIPLIER` | `5` |
+| `ETH1_FOLLOW_DISTANCE` | `uint64(2**10)` (= 1,024) |
+| `MAX_COMMITTEES_PER_SLOT` | `uint64(2**6)` (= 64) |
+| `TARGET_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) |
+| `MAX_VALIDATORS_PER_COMMITTEE` | `uint64(2**11)` (= 2,048) |
+| `MIN_PER_EPOCH_CHURN_LIMIT` | `uint64(2**2)` (= 4) |
+| `CHURN_LIMIT_QUOTIENT` | `uint64(2**16)` (= 65,536) |
+| `SHUFFLE_ROUND_COUNT` | `uint64(90)` |
+| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `uint64(2**14)` (= 16,384) |
+| `MIN_GENESIS_TIME` | `uint64(1578009600)` (Jan 3, 2020) |
+| `HYSTERESIS_QUOTIENT` | `uint64(4)` |
+| `HYSTERESIS_DOWNWARD_MULTIPLIER` | `uint64(1)` |
+| `HYSTERESIS_UPWARD_MULTIPLIER` | `uint64(5)` |
 
 - For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.)
 
@@ -218,37 +218,37 @@ The following values are (non-configurable) constants used throughout the specif
 
 | Name | Value | Unit | Duration |
 | - | - | :-: | :-: |
-| `GENESIS_DELAY` | `172800` | seconds | 2 days |
-| `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds |
-| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds |
-| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds |
-| `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes |
-| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes |
-| `MAX_SEED_LOOKAHEAD` | `2**2` (= 4) | epochs | 25.6 minutes |
-| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes |
-| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**5` (= 32) | epochs | ~3.4 hours |
-| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours |
-| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours |
-| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours |
+| `GENESIS_DELAY` | `uint64(172800)` | seconds | 2 days |
+| `SECONDS_PER_SLOT` | `uint64(12)` | seconds | 12 seconds |
+| `SECONDS_PER_ETH1_BLOCK` | `uint64(14)` | seconds | 14 seconds |
+| `MIN_ATTESTATION_INCLUSION_DELAY` | `uint64(2**0)` (= 1) | slots | 12 seconds |
+| `SLOTS_PER_EPOCH` | `uint64(2**5)` (= 32) | slots | 6.4 minutes |
+| `MIN_SEED_LOOKAHEAD` | `uint64(2**0)` (= 1) | epochs | 6.4 minutes |
+| `MAX_SEED_LOOKAHEAD` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes |
+| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes |
+| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `uint64(2**5)` (= 32) | epochs | ~3.4 hours |
+| `SLOTS_PER_HISTORICAL_ROOT` | `uint64(2**13)` (= 8,192) | slots | ~27 hours |
+| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `uint64(2**8)` (= 256) | epochs | ~27 hours |
+| `SHARD_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours |
 
 ### State list lengths
 
 | Name | Value | Unit | Duration |
 | - | - | :-: | :-: |
-| `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years |
-| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days |
-| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~52,262 years |
-| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validators |
+| `EPOCHS_PER_HISTORICAL_VECTOR` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 years |
+| `EPOCHS_PER_SLASHINGS_VECTOR` | `uint64(2**13)` (= 8,192) | epochs | ~36 days |
+| `HISTORICAL_ROOTS_LIMIT` | `uint64(2**24)` (= 16,777,216) | historical roots | ~52,262 years |
+| `VALIDATOR_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | validators |
 
 ### Rewards and penalties
 
 | Name | Value |
 | - | - |
-| `BASE_REWARD_FACTOR` | `2**6` (= 64) |
-| `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) |
-| `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) |
-| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) |
-| `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) |
+| `BASE_REWARD_FACTOR` | `uint64(2**6)` (= 64) |
+| `WHISTLEBLOWER_REWARD_QUOTIENT` | `uint64(2**9)` (= 512) |
+| `PROPOSER_REWARD_QUOTIENT` | `uint64(2**3)` (= 8) |
+| `INACTIVITY_PENALTY_QUOTIENT` | `uint64(2**24)` (= 16,777,216) |
+| `MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**5)` (= 32) |
 
 - The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12` epochs (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`.
 
@@ -587,7 +587,7 @@ def bytes_to_uint64(data: bytes) -> uint64:
     """
     Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian.
     """
-    return int.from_bytes(data, ENDIANNESS)
+    return uint64(int.from_bytes(data, ENDIANNESS))
 ```
 
 ### Crypto
@@ -735,7 +735,7 @@ def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) ->
             + uint_to_bytes(uint8(current_round))
             + uint_to_bytes(uint32(position // 256))
         )
-        byte = source[(position % 256) // 8]
+        byte = uint8(source[(position % 256) // 8])
         bit = (byte >> (position % 8)) % 2
         index = flip if bit else index
 
@@ -751,9 +751,10 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex]
     """
     assert len(indices) > 0
     MAX_RANDOM_BYTE = 2**8 - 1
-    i = 0
+    i = uint64(0)
+    total = uint64(len(indices))
     while True:
-        candidate_index = indices[compute_shuffled_index(i % len(indices), len(indices), seed)]
+        candidate_index = indices[compute_shuffled_index(i % total, total, seed)]
         random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32]
         effective_balance = state.validators[candidate_index].effective_balance
         if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte:
@@ -773,7 +774,7 @@ def compute_committee(indices: Sequence[ValidatorIndex],
     """
     start = (len(indices) * index) // count
     end = (len(indices) * (index + 1)) // count
-    return [indices[compute_shuffled_index(i, len(indices), seed)] for i in range(start, end)]
+    return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)]
 ```
 
 #### `compute_epoch_at_slot`
@@ -932,7 +933,7 @@ def get_validator_churn_limit(state: BeaconState) -> uint64:
     Return the validator churn limit for the current epoch.
     """
     active_validator_indices = get_active_validator_indices(state, get_current_epoch(state))
-    return max(MIN_PER_EPOCH_CHURN_LIMIT, len(active_validator_indices) // CHURN_LIMIT_QUOTIENT)
+    return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT)
 ```
 
 #### `get_seed`
@@ -953,9 +954,9 @@ def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64:
     """
     Return the number of committees in each slot for the given ``epoch``.
     """
-    return max(1, min(
+    return max(uint64(1), min(
         MAX_COMMITTEES_PER_SLOT,
-        len(get_active_validator_indices(state, epoch)) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE,
+        uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE,
     ))
 ```
 
@@ -1314,7 +1315,7 @@ def process_justification_and_finalization(state: BeaconState) -> None:
 
     # Process justifications
     state.previous_justified_checkpoint = state.current_justified_checkpoint
-    state.justification_bits[1:] = state.justification_bits[:-1]
+    state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1]
     state.justification_bits[0] = 0b0
     matching_target_attestations = get_matching_target_attestations(state, previous_epoch)  # Previous epoch
     if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2:

diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md
@@ -102,22 +102,22 @@ _The block for `anchor_root` is incorrectly initialized to the block header, rat
 
 ```python
 def get_forkchoice_store(anchor_state: BeaconState) -> Store:
-    anchor_block_header = anchor_state.latest_block_header.copy()
+    anchor_block_header = copy(anchor_state.latest_block_header)
     if anchor_block_header.state_root == Bytes32():
         anchor_block_header.state_root = hash_tree_root(anchor_state)
     anchor_root = hash_tree_root(anchor_block_header)
     anchor_epoch = get_current_epoch(anchor_state)
     justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root)
     finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root)
     return Store(
-        time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot,
+        time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot),
         genesis_time=anchor_state.genesis_time,
         justified_checkpoint=justified_checkpoint,
         finalized_checkpoint=finalized_checkpoint,
         best_justified_checkpoint=justified_checkpoint,
         blocks={anchor_root: anchor_block_header},
-        block_states={anchor_root: anchor_state.copy()},
-        checkpoint_states={justified_checkpoint: anchor_state.copy()},
+        block_states={anchor_root: copy(anchor_state)},
+        checkpoint_states={justified_checkpoint: copy(anchor_state)},
     )
 ```
 
@@ -302,7 +302,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None:
 def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None:
     # Store target checkpoint state if not yet seen
     if target not in store.checkpoint_states:
-        base_state = store.block_states[target.root].copy()
+        base_state = copy(store.block_states[target.root])
         if base_state.slot < compute_start_slot_at_epoch(target.epoch):
             process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
         store.checkpoint_states[target] = base_state
@@ -348,7 +348,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # Parent block must be known
     assert block.parent_root in store.block_states
     # Make a copy of the state to avoid mutability issues
-    pre_state = store.block_states[block.parent_root].copy()
+    pre_state = copy(store.block_states[block.parent_root])
     # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
     assert get_current_slot(store) >= block.slot