Simplify build system for computed constants

pysetup/helpers.py

@@ -25,7 +25,7 @@ def collect_prev_forks(fork: str) -> list[str]:
 def requires_mypy_type_ignore(value: str) -> bool:
     return (
         value.startswith(('ByteVector'))
-        or (value.startswith(('Vector')) and 'floorlog2' in value)
+        or (value.startswith(('Vector')) and any(k in value for k in ['ceillog2', 'floorlog2']))
     )
 
 
@@ -41,15 +41,19 @@ def objects_to_spec(preset_name: str,
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
     """
-    new_type_definitions = (
-        '\n\n'.join(
-            [
-                f"class {key}({value}):\n    pass\n" if not requires_mypy_type_ignore(value)
-                else f"class {key}({value}):  # type: ignore\n    pass\n"
-                for key, value in spec_object.custom_types.items()
-            ]
+    def gen_new_type_definitions(custom_types: Dict[str, str]) -> str:
+        return (
+            '\n\n'.join(
+                [
+                    f"class {key}({value}):\n    pass\n" if not requires_mypy_type_ignore(value)
+                    else f"class {key}({value}):  # type: ignore\n    pass\n"
+                    for key, value in custom_types.items()
+                ]
+            )
         )
-    )
+
+    new_type_definitions = gen_new_type_definitions(spec_object.custom_types)
+    preset_dep_new_type_definitions = gen_new_type_definitions(spec_object.preset_dep_custom_types)
 
     # Collect builders with the reversed previous forks
     # e.g. `[bellatrix, altair, phase0]` -> `[phase0, altair, bellatrix]`
@@ -115,7 +119,6 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
 
     # Merge all constant objects
     hardcoded_ssz_dep_constants =         reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
-    hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
     hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
     # Concatenate all strings
     imports =              reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name)  ).strip("\n"), builders, "")
@@ -135,25 +138,26 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     filtered_hardcoded_func_dep_presets = {k: v for k, v in hardcoded_func_dep_presets.items() if k not in deprecate_presets}
 
     constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items())
+    preset_dep_constant_vars_spec = '# Preset computed constants\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_dep_constant_vars.items())
     preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items())
     ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
     ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), filtered_ssz_dep_constants))
-    custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
     func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), filtered_hardcoded_func_dep_presets))
     spec_strs = [
         imports,
         preparations,
         f"fork = \'{fork}\'\n",
         # The helper functions that some SSZ containers require. Need to be defined before `custom_type_dep_constants`
         CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS,
-        # The constants that some SSZ containers require. Need to be defined before `new_type_definitions`
-        custom_type_dep_constants,
         # The constants that some SSZ containers require. Need to be defined before `constants_spec`
         ssz_dep_constants,
         new_type_definitions,
         constant_vars_spec,
+        # The presets that some SSZ types require. Need to be defined before `preset_dep_new_type_definitions`
         preset_vars_spec,
+        preset_dep_constant_vars_spec,
+        preset_dep_new_type_definitions,
         config_spec,
         # Custom classes which are not required to be SSZ containers.
         classes,
@@ -237,7 +241,9 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     protocols = combine_protocols(spec0.protocols, spec1.protocols)
     functions = combine_dicts(spec0.functions, spec1.functions)
     custom_types = combine_dicts(spec0.custom_types, spec1.custom_types)
+    preset_dep_custom_types = combine_dicts(spec0.preset_dep_custom_types, spec1.preset_dep_custom_types)
     constant_vars = combine_dicts(spec0.constant_vars, spec1.constant_vars)
+    preset_dep_constant_vars = combine_dicts(spec0.preset_dep_constant_vars, spec1.preset_dep_constant_vars)
     preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
     config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
     ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
@@ -248,7 +254,9 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
         functions=functions,
         protocols=protocols,
         custom_types=custom_types,
+        preset_dep_custom_types=preset_dep_custom_types,
         constant_vars=constant_vars,
+        preset_dep_constant_vars=preset_dep_constant_vars,
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,

pysetup/spec_builders/base.py

@@ -47,13 +47,6 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
         """
         return {}
 
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:  # TODO
-        """
-        The constants that are required for custom types.
-        """
-        return {}
-
     @classmethod
     def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
         return {}

pysetup/spec_builders/bellatrix.py

@@ -62,10 +62,3 @@ def verify_and_notify_new_payload(self: ExecutionEngine,
 
 
 EXECUTION_ENGINE = NoopExecutionEngine()"""
-
-
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
-        return {
-            'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value,
-        }

pysetup/spec_builders/deneb.py

@@ -80,15 +80,6 @@ def verify_and_notify_new_payload(self: ExecutionEngine,
 EXECUTION_ENGINE = NoopExecutionEngine()"""
 
 
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
-        return {
-            'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
-            'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
-            'MAX_BLOBS_PER_BLOCK': spec_object.config_vars['MAX_BLOBS_PER_BLOCK'].value,
-            'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
-        }
-
     @classmethod
     def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
         return {

pysetup/spec_builders/eip6800.py

@@ -13,9 +13,3 @@ def imports(cls, preset_name: str):
 from eth2spec.deneb import {preset_name} as deneb
 from eth2spec.utils.ssz.ssz_typing import Bytes31
 '''
-
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
-        return {
-            'MAX_STEMS': spec_object.preset_vars['MAX_STEMS'].value,
-        }

pysetup/spec_builders/eip7441.py

@@ -14,16 +14,6 @@ def imports(cls, preset_name: str):
 import json
 '''
 
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
-        # Necessary for custom types `WhiskShuffleProof` and `WhiskTrackerProof`
-        return {
-            'MAX_SHUFFLE_PROOF_SIZE': spec_object.preset_vars['MAX_SHUFFLE_PROOF_SIZE'].value,
-            'MAX_OPENING_PROOF_SIZE': spec_object.preset_vars['MAX_OPENING_PROOF_SIZE'].value,
-            'VALIDATORS_PER_SHUFFLE': spec_object.preset_vars['VALIDATORS_PER_SHUFFLE'].value,
-            'CURDLEPROOFS_N_BLINDERS': spec_object.preset_vars['CURDLEPROOFS_N_BLINDERS'].value,
-        }
-
     @classmethod
     def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
         constants = {

pysetup/spec_builders/eip7732.py

@@ -23,15 +23,6 @@ def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex:
     return o'''
 
 
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
-        return {
-            'PTC_SIZE': spec_object.preset_vars['PTC_SIZE'].value,
-            'MAX_PAYLOAD_ATTESTATIONS': spec_object.preset_vars['MAX_PAYLOAD_ATTESTATIONS'].value,
-            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732':
-                spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732'].value,
-        }
-
     @classmethod
     def deprecate_constants(cls) -> Set[str]:
         return set([

pysetup/spec_builders/fulu.py

@@ -50,14 +50,6 @@ def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSide
     return []
 """
 
-    @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
-        return {
-            'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
-            'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value,
-            'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value,
-        }
-
     @classmethod
     def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
         return {

pysetup/typing.py

@@ -18,7 +18,9 @@ class SpecObject(NamedTuple):
     functions: Dict[str, str]
     protocols: Dict[str, ProtocolDefinition]
     custom_types: Dict[str, str]
+    preset_dep_custom_types: Dict[str, str]  # the types that depend on presets
     constant_vars: Dict[str, VariableDefinition]
+    preset_dep_constant_vars: Dict[str, VariableDefinition]
     preset_vars: Dict[str, VariableDefinition]
     config_vars: Dict[str, VariableDefinition]
     ssz_dep_constants: Dict[str, str]  # the constants that depend on ssz_objects

setup.py

@@ -175,12 +175,33 @@ def _parse_value(name: str, typed_value: str, type_hint: Optional[str] = None) -
     return VariableDefinition(type_name=type_name, value=typed_value[i+1:-1], comment=comment, type_hint=type_hint)
 
 
-def _update_constant_vars_with_kzg_setups(constant_vars, preset_name):
+def _update_constant_vars_with_kzg_setups(constant_vars, preset_dep_constant_vars, preset_name):
     comment = "noqa: E501"
     kzg_setups = ALL_KZG_SETUPS[preset_name]
-    constant_vars['KZG_SETUP_G1_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G1_MONOMIAL'].value, str(kzg_setups[0]), comment, None)
-    constant_vars['KZG_SETUP_G1_LAGRANGE'] = VariableDefinition(constant_vars['KZG_SETUP_G1_LAGRANGE'].value, str(kzg_setups[1]), comment, None)
-    constant_vars['KZG_SETUP_G2_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G2_MONOMIAL'].value, str(kzg_setups[2]), comment, None)
+    preset_dep_constant_vars['KZG_SETUP_G1_MONOMIAL'] = VariableDefinition(
+        preset_dep_constant_vars['KZG_SETUP_G1_MONOMIAL'].value,
+        str(kzg_setups[0]),
+        comment, None
+    )
+    preset_dep_constant_vars['KZG_SETUP_G1_LAGRANGE'] = VariableDefinition(
+        preset_dep_constant_vars['KZG_SETUP_G1_LAGRANGE'].value,
+        str(kzg_setups[1]),
+        comment, None
+    )
+    constant_vars['KZG_SETUP_G2_MONOMIAL'] = VariableDefinition(
+        constant_vars['KZG_SETUP_G2_MONOMIAL'].value,
+        str(kzg_setups[2]),
+        comment, None
+    )
+
+
+def _update_constant_vars_with_curdleproofs_crs(constant_vars, preset_dep_constant_vars, preset_name):
+    comment = "noqa: E501"
+    constant_vars['CURDLEPROOFS_CRS'] = VariableDefinition(
+        None,
+        'curdleproofs.CurdleproofsCrs.from_json(json.dumps(' + str(ALL_CURDLEPROOFS_CRS[str(preset_name)]).replace('0x', '') + '))',
+        comment, None
+    )
 
 
 @lru_cache(maxsize=None)
@@ -192,13 +213,14 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
     functions: Dict[str, str] = {}
     protocols: Dict[str, ProtocolDefinition] = {}
     constant_vars: Dict[str, VariableDefinition] = {}
+    preset_dep_constant_vars: Dict[str, VariableDefinition] = {}
     preset_vars: Dict[str, VariableDefinition] = {}
     config_vars: Dict[str, VariableDefinition] = {}
     ssz_dep_constants: Dict[str, str] = {}
     func_dep_presets: Dict[str, str] = {}
     ssz_objects: Dict[str, str] = {}
     dataclasses: Dict[str, str] = {}
-    custom_types: Dict[str, str] = {}
+    all_custom_types: Dict[str, str] = {}
 
     with open(file_name) as source_file:
         document = parse_markdown(source_file.read())
@@ -278,7 +300,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                     if not _is_constant_id(name):
                         # Check for short type declarations
                         if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")):
-                            custom_types[name] = value
+                            all_custom_types[name] = value
                         continue
 
                     if value.startswith("get_generalized_index"):
@@ -297,7 +319,10 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                         if name in ('ENDIANNESS', 'KZG_ENDIANNESS'):
                             # Deal with mypy Literal typing check
                             value_def = _parse_value(name, value, type_hint='Final')
-                        constant_vars[name] = value_def
+                        if any(k in value for k in preset) or any(k in value for k in preset_dep_constant_vars):
+                            preset_dep_constant_vars[name] = value_def
+                        else:
+                            constant_vars[name] = value_def
 
         elif isinstance(child, LinkRefDef):
             comment = _get_eth2_spec_comment(child)
@@ -306,20 +331,26 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
     # Load KZG trusted setup from files
     if any('KZG_SETUP' in name for name in constant_vars):
-        _update_constant_vars_with_kzg_setups(constant_vars, preset_name)
+        _update_constant_vars_with_kzg_setups(constant_vars, preset_dep_constant_vars, preset_name)
 
     if any('CURDLEPROOFS_CRS' in name for name in constant_vars):
-        constant_vars['CURDLEPROOFS_CRS'] = VariableDefinition(
-            None,
-            'curdleproofs.CurdleproofsCrs.from_json(json.dumps(' + str(ALL_CURDLEPROOFS_CRS[str(preset_name)]).replace('0x', '') + '))',
-            "noqa: E501", None
-        )
+        _update_constant_vars_with_curdleproofs_crs(constant_vars, preset_dep_constant_vars, preset_name)
+
+    custom_types: Dict[str, str] = {}
+    preset_dep_custom_types: Dict[str, str] = {}
+    for name, value in all_custom_types.items():
+        if any(k in value for k in preset) or any(k in value for k in preset_dep_constant_vars):
+            preset_dep_custom_types[name] = value
+        else:
+            custom_types[name] = value
 
     return SpecObject(
         functions=functions,
         protocols=protocols,
         custom_types=custom_types,
+        preset_dep_custom_types=preset_dep_custom_types,
         constant_vars=constant_vars,
+        preset_dep_constant_vars=preset_dep_constant_vars,
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,
@@ -377,7 +408,10 @@ def build_spec(fork: str,
     new_objects = {}
     while OrderedDict(new_objects) != OrderedDict(class_objects):
         new_objects = copy.deepcopy(class_objects)
-        dependency_order_class_objects(class_objects, spec_object.custom_types)
+        dependency_order_class_objects(
+            class_objects,
+            spec_object.custom_types | spec_object.preset_dep_custom_types,
+        )
 
     return objects_to_spec(preset_name, spec_object, fork, class_objects)
 

specs/fulu/polynomial-commitments-sampling.md

@@ -746,7 +746,7 @@ def recover_cells_and_kzg_proofs(cell_indices: Sequence[CellIndex],
     # Check we have the same number of cells and indices
     assert len(cell_indices) == len(cells)
     # Check we have enough cells to be able to perform the reconstruction
-    assert CELLS_PER_EXT_BLOB / 2 <= len(cell_indices) <= CELLS_PER_EXT_BLOB
+    assert CELLS_PER_EXT_BLOB // 2 <= len(cell_indices) <= CELLS_PER_EXT_BLOB
     # Check for duplicates
     assert len(cell_indices) == len(set(cell_indices))
     # Check that the cell indices are within bounds