diff --git a/packages/ssz/package.tgz b/packages/ssz/package.tgz new file mode 100644 index 00000000..0cc5135d Binary files /dev/null and b/packages/ssz/package.tgz differ diff --git a/packages/ssz/src/index.ts b/packages/ssz/src/index.ts index b0c6d077..7d33d5de 100644 --- a/packages/ssz/src/index.ts +++ b/packages/ssz/src/index.ts @@ -15,6 +15,8 @@ export {OptionalType} from "./type/optional"; export {VectorBasicType} from "./type/vectorBasic"; export {VectorCompositeType} from "./type/vectorComposite"; export {ListUintNum64Type} from "./type/listUintNum64"; +export {StableContainerType} from "./type/stableContainer"; +export {ProfileType} from "./type/profile"; // Base types export {ArrayType} from "./type/array"; diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 5ae7e2bb..7c5f9baf 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -15,6 +15,11 @@ import {CompositeType, isCompositeType} from "./composite"; import {addLengthNode, getLengthFromRootNode} from "./arrayBasic"; /* eslint-disable @typescript-eslint/member-ordering */ +export type NonOptionalType> = T extends OptionalType ? U : T; +export type NonOptionalFields>> = { + [K in keyof Fields]: NonOptionalType; +}; + export type OptionalOpts = { typeName?: string; }; @@ -258,3 +263,11 @@ export class OptionalType> extends CompositeTy return this.elementType.equals(a, b); } } + +export function isOptionalType(type: Type): type is OptionalType> { + return type instanceof OptionalType; +} + +export function toNonOptionalType>(type: T): NonOptionalType { + return (isOptionalType(type) ? type.elementType : type) as NonOptionalType; +} diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts new file mode 100644 index 00000000..f9469fe0 --- /dev/null +++ b/packages/ssz/src/type/profile.ts @@ -0,0 +1,670 @@ +import { + Node, + getNodesAtDepth, + subtreeFillToContents, + Tree, + Gindex, + toGindex, + concatGindices, + getNode, + BranchNode, + zeroHash, + zeroNode, +} from "@chainsafe/persistent-merkle-tree"; +import {ValueWithCachedPermanentRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; +import {Require} from "../util/types"; +import {namedClass} from "../util/named"; +import {Type, ValueOf} from "./abstract"; +import {CompositeType, ByteViews, CompositeTypeAny} from "./composite"; +import { + getProfileTreeViewClass, + ValueOfFields, + FieldEntry, + ContainerTreeViewType, + ContainerTreeViewTypeConstructor, + computeSerdesData, +} from "../view/profile"; +import { + getProfileTreeViewDUClass, + ContainerTreeViewDUType, + ContainerTreeViewDUTypeConstructor, +} from "../viewDU/profile"; +import {Case} from "../util/strings"; +import {BitArray} from "../value/bitArray"; +import {mixInActiveFields, setActiveFields} from "./stableContainer"; +import {NonOptionalFields, isOptionalType, toNonOptionalType} from "./optional"; +/* eslint-disable @typescript-eslint/member-ordering */ + +type BytesRange = {start: number; end: number}; + +export type ProfileOptions> = { + typeName?: string; + jsonCase?: KeyCase; + casingMap?: CasingMap; + cachePermanentRootStruct?: boolean; + getProfileTreeViewClass?: typeof getProfileTreeViewClass; + getProfileTreeViewDUClass?: typeof getProfileTreeViewDUClass; +}; + +export type KeyCase = + | "eth2" + | "snake" + | "constant" + | "camel" + | "header" + //Same as squish + | "pascal"; + +type CasingMap> = Partial<{[K in keyof Fields]: string}>; + +/** + * Profile: ordered heterogeneous collection of values that inherits merkleization from a base stable container + * - EIP: https://eips.ethereum.org/EIPS/eip-7495 + * - No reordering of fields for merkleization + */ +export class ProfileType>> extends CompositeType< + ValueOfFields, + ContainerTreeViewType, + ContainerTreeViewDUType +> { + readonly typeName: string; + readonly depth: number; + readonly maxChunkCount: number; + readonly fixedSize: number | null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = false; + readonly isViewMutable = true; + readonly activeFields: BitArray; + + // Precomputed data for faster serdes + readonly fieldsEntries: FieldEntry>[]; + /** End of fixed section of serialized Container */ + protected readonly fieldsGindex: Record; + protected readonly jsonKeyToFieldName: Record; + + /** Cached TreeView constuctor with custom prototype for this Type's properties */ + protected readonly TreeView: ContainerTreeViewTypeConstructor; + protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; + private optionalFieldsCount: number; + + constructor(readonly fields: Fields, activeFields: BitArray, readonly opts?: ProfileOptions) { + super(); + + // Render detailed typeName. Consumers should overwrite since it can get long + this.typeName = opts?.typeName ?? renderContainerTypeName(fields); + + if (activeFields.getTrueBitIndexes().length !== Object.keys(fields).length) { + throw new Error("activeFields must have the same number of true bits as fields"); + } + + this.activeFields = activeFields; + this.maxChunkCount = this.activeFields.bitLen; + this.depth = maxChunksToDepth(this.maxChunkCount) + 1; + + // Precalculated data for faster serdes + this.fieldsEntries = []; + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + this.optionalFieldsCount = 0; + for (let i = 0, fieldIx = 0; i < this.activeFields.bitLen; i++) { + if (!this.activeFields.get(i)) { + continue; + } + + const fieldName = fieldNames[fieldIx++]; + const fieldType = fields[fieldName]; + const optional = isOptionalType(fieldType); + this.fieldsEntries.push({ + fieldName, + fieldType: toNonOptionalType(fieldType), + jsonKey: precomputeJsonKey(fieldName, opts?.casingMap, opts?.jsonCase), + gindex: toGindex(this.depth, BigInt(i)), + chunkIndex: i, + optional, + }); + + if (optional) { + this.optionalFieldsCount++; + } + } + + if (this.fieldsEntries.length === 0) { + throw Error("Container must have > 0 fields"); + } + + // Precalculate for Proofs API + this.fieldsGindex = {} as Record; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, chunkIndex} = this.fieldsEntries[i]; + this.fieldsGindex[fieldName] = toGindex(this.depth, BigInt(chunkIndex)); + } + + // To resolve JSON paths in fieldName notation and jsonKey notation + this.jsonKeyToFieldName = {}; + for (const {fieldName, jsonKey} of this.fieldsEntries) { + this.jsonKeyToFieldName[jsonKey] = fieldName; + } + + const {minLen, maxLen, fixedSize} = precomputeSizes(fields); + this.minSize = minLen; + this.maxSize = maxLen; + this.fixedSize = fixedSize; + + // TODO: This options are necessary for ContainerNodeStruct to override this. + // Refactor this constructor to allow customization without pollutin the options + this.TreeView = opts?.getProfileTreeViewClass?.(this) ?? getProfileTreeViewClass(this); + this.TreeViewDU = opts?.getProfileTreeViewDUClass?.(this) ?? getProfileTreeViewDUClass(this); + } + + static named>>( + fields: Fields, + activeFields: BitArray, + opts: Require, "typeName"> + ): ProfileType { + return new (namedClass(ProfileType, opts.typeName))(fields, activeFields, opts); + } + + defaultValue(): ValueOfFields { + const value = {} as ValueOfFields; + for (const {fieldName, fieldType, optional} of this.fieldsEntries) { + value[fieldName] = (optional ? null : fieldType.defaultValue()) as ValueOf; + } + return value; + } + + getView(tree: Tree): ContainerTreeViewType { + return new this.TreeView(this, tree); + } + + getViewDU(node: Node, cache?: unknown): ContainerTreeViewDUType { + return new this.TreeViewDU(this, node, cache); + } + + cacheOfViewDU(view: ContainerTreeViewDUType): unknown { + return view.cache; + } + + commitView(view: ContainerTreeViewType): Node { + return view.node; + } + + commitViewDU(view: ContainerTreeViewDUType): Node { + view.commit(); + return view.node; + } + + // Serialization + deserialization + // ------------------------------- + // Containers can mix fixed length and variable length data. + // + // Fixed part Variable part + // [field1 offset][field2 data ][field1 data ] + // [0x000000c] [0xaabbaabbaabbaabb][0xffffffffffffffffffffffff] + + value_serializedSize(value: ValueOfFields): number { + let totalSize = Math.ceil(this.optionalFieldsCount / 8); + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && value[fieldName] == null) { + continue; + } + // Offset (4 bytes) + size + totalSize += + fieldType.fixedSize === null ? 4 + fieldType.value_serializedSize(value[fieldName]) : fieldType.fixedSize; + } + return totalSize; + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOfFields): number { + const optionalFields = BitArray.fromBitLen(this.optionalFieldsCount); + let optionalIndex = 0; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, optional} = this.fieldsEntries[i]; + if (optional) { + optionalFields.set(optionalIndex++, value[fieldName] !== null); + } + } + + output.uint8Array.set(optionalFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(optionalFields, this.fieldsEntries); + + const optionalFieldsLen = optionalFields.uint8Array.length; + let fixedIndex = offset + optionalFieldsLen; + let variableIndex = offset + fixedEnd + optionalFieldsLen; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + // skip optional fields with nullish values + if (optional && value[fieldName] == null) { + continue; + } + + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - optionalFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.value_serializeToBytes(output, variableIndex, value[fieldName]); + } else { + fixedIndex = fieldType.value_serializeToBytes(output, fixedIndex, value[fieldName]); + } + } + return variableIndex; + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + const {optionalFields, fieldRanges} = this.getFieldRanges(data, start, end); + const value = {} as {[K in keyof Fields]: unknown}; + const optionalFieldsLen = optionalFields.uint8Array.length; + start += optionalFieldsLen; + + let optionalIndex = 0; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && !optionalFields.get(optionalIndex++)) { + value[fieldName] = null; + continue; + } + const fieldRange = fieldRanges[i]; + value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + return value as ValueOfFields; + } + + tree_serializedSize(node: Node): number { + let totalSize = Math.ceil(this.optionalFieldsCount / 8); + const nodes = getNodesAtDepth(node, this.depth, 0, this.activeFields.bitLen) as Node[]; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex, optional} = this.fieldsEntries[i]; + const node = nodes[chunkIndex]; + // zeroNode() means optional field is null, it's different from a node with all zeros + if (optional && node === zeroNode(0)) { + continue; + } + // Offset (4 bytes) + size + totalSize += fieldType.fixedSize === null ? 4 + fieldType.tree_serializedSize(node) : fieldType.fixedSize; + } + return totalSize; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + const optionalFields = BitArray.fromBitLen(this.optionalFieldsCount); + const optionalFieldsLen = optionalFields.uint8Array.length; + + const nodes = getNodesAtDepth(node, this.depth, 0, this.activeFields.bitLen); + let optionalIndex = -1; + if (this.optionalFieldsCount > 0) { + // 1st loop to compute optional fields + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {chunkIndex, optional} = this.fieldsEntries[i]; + const node = nodes[chunkIndex]; + if (optional) { + optionalIndex++; + if (node !== zeroNode(0)) { + optionalFields.set(optionalIndex, true); + } + } + } + } + + output.uint8Array.set(optionalFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(optionalFields, this.fieldsEntries); + let fixedIndex = offset + optionalFieldsLen; + let variableIndex = offset + fixedEnd + optionalFieldsLen; + + // 2nd loop to serialize fields + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex, optional} = this.fieldsEntries[i]; + const node = nodes[chunkIndex]; + if (optional && node === zeroNode(0)) { + continue; + } + + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - optionalFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + return variableIndex; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const {optionalFields, fieldRanges} = this.getFieldRanges(data, start, end); + const nodes = new Array(this.activeFields.bitLen).fill(zeroNode(0)); + const optionalFieldsLen = optionalFields.uint8Array.length; + start += optionalFieldsLen; + + let optionalIndex = -1; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex, optional} = this.fieldsEntries[i]; + if (optional) { + optionalIndex++; + if (!optionalFields.get(optionalIndex)) { + continue; + } + } + + const fieldRange = fieldRanges[i]; + nodes[chunkIndex] = fieldType.tree_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + const root = new BranchNode(subtreeFillToContents(nodes, this.depth - 1), zeroNode(0)); + return setActiveFields(root, this.activeFields); + } + + // Merkleization + hashTreeRoot(value: ValueOfFields): Uint8Array { + // Return cached mutable root if any + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + return cachedRoot; + } + } + + const root = mixInActiveFields(super.hashTreeRoot(value), this.activeFields); + + if (this.cachePermanentRootStruct) { + (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + } + + return root; + } + + protected getRoots(struct: ValueOfFields): Uint8Array[] { + const roots = new Array(this.activeFields.bitLen).fill(zeroHash(0)); + + // already asserted that # of active fields in bitvector === # of fields + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, chunkIndex, optional} = this.fieldsEntries[i]; + if (optional && struct[fieldName] == null) { + continue; + } + roots[chunkIndex] = fieldType.hashTreeRoot(struct[fieldName]); + } + + return roots; + } + + // Proofs + + /** INTERNAL METHOD: For view's API, create proof from a tree */ + + getPropertyGindex(prop: string): Gindex | null { + const gindex = this.fieldsGindex[prop] ?? this.fieldsGindex[this.jsonKeyToFieldName[prop]]; + if (gindex === undefined) throw Error(`Unknown container property ${prop}`); + return gindex; + } + + getPropertyType(prop: string): Type { + const type = this.fields[prop] ?? this.fields[this.jsonKeyToFieldName[prop]]; + if (type === undefined) throw Error(`Unknown container property ${prop}`); + return type; + } + + getIndexProperty(index: number): string | null { + if (index >= this.fieldsEntries.length) { + return null; + } + return this.fieldsEntries[index].fieldName as string; + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const gindices: Gindex[] = []; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + const fieldGindex = this.fieldsGindex[fieldName]; + const fieldGindexFromRoot = concatGindices([rootGindex, fieldGindex]); + + if (fieldType.isBasic) { + gindices.push(fieldGindexFromRoot); + } else { + const compositeType = fieldType as unknown as CompositeTypeAny; + if (fieldType.fixedSize === null) { + if (!rootNode) { + throw new Error("variable type requires tree argument to get leaves"); + } + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot, getNode(rootNode, fieldGindex))); + } else { + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot)); + } + } + } + + return gindices; + } + + // JSON + + fromJson(json: unknown): ValueOfFields { + if (typeof json !== "object") { + throw Error("JSON must be of type object"); + } + if (json === null) { + throw Error("JSON must not be null"); + } + + const value = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey} = this.fieldsEntries[i]; + const jsonValue = (json as Record)[jsonKey]; + if (jsonValue === undefined) { + throw Error(`JSON expected key ${jsonKey} is undefined`); + } + value[fieldName] = fieldType.fromJson(jsonValue) as ValueOf; + } + + return value; + } + + toJson(value: ValueOfFields): Record { + const json: Record = {}; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey} = this.fieldsEntries[i]; + json[jsonKey] = fieldType.toJson(value[fieldName]); + } + + return json; + } + + clone(value: ValueOfFields): ValueOfFields { + const newValue = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + newValue[fieldName] = fieldType.clone(value[fieldName]) as ValueOf; + } + + return newValue; + } + + equals(a: ValueOfFields, b: ValueOfFields): boolean { + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + if (!fieldType.equals(a[fieldName], b[fieldName])) { + return false; + } + } + + return true; + } + + /** + * Deserializer helper: Returns the bytes ranges of all fields, both variable and fixed size. + * Fields may not be contiguous in the serialized bytes, so the returned ranges are [start, end]. + * - For fixed size fields re-uses the pre-computed values this.fieldRangesFixedLen + * - For variable size fields does a first pass over the fixed section to read offsets + * - offsets are relative to the start of serialized active fields, after the Bitvector[N] + */ + getFieldRanges(data: ByteViews, start: number, end: number): {optionalFields: BitArray; fieldRanges: BytesRange[]} { + const optionalFieldsByteLen = Math.ceil(this.optionalFieldsCount / 8); + const optionalFields = new BitArray( + data.uint8Array.subarray(start, start + optionalFieldsByteLen), + this.optionalFieldsCount + ); + + const {variableOffsetsPosition, fixedEnd, fieldRangesFixedLen, isFixedLen} = computeSerdesData( + optionalFields, + this.fieldsEntries + ); + + if (variableOffsetsPosition.length === 0) { + // Validate fixed length container + const size = end - start; + if (size !== fixedEnd + optionalFieldsByteLen) { + throw Error( + `${this.typeName} size ${size} not equal fixed end plus optionalFieldsByteLen ${ + fixedEnd + optionalFieldsByteLen + }` + ); + } + + return {optionalFields, fieldRanges: fieldRangesFixedLen}; + } + + // Read offsets in one pass + const offsets = readVariableOffsets( + data.dataView, + start, + end, + optionalFieldsByteLen, + fixedEnd, + variableOffsetsPosition + ); + offsets.push(end - start - optionalFieldsByteLen); // The offsets are relative to the start of serialized optional fields + + // Merge fieldRangesFixedLen + offsets in one array + let variableIdx = 0; + let fixedIdx = 0; + const fieldRanges = new Array(isFixedLen.length); + + for (let i = 0; i < isFixedLen.length; i++) { + if (isFixedLen[i]) { + // push from fixLen ranges ++ + fieldRanges[i] = fieldRangesFixedLen[fixedIdx++]; + } else { + // push from varLen ranges ++ + fieldRanges[i] = {start: offsets[variableIdx], end: offsets[variableIdx + 1]}; + variableIdx++; + } + } + + return {optionalFields, fieldRanges}; + } +} + +/** + * Returns the byte ranges of all variable size fields. + * Offsets are relative to the start of serialized active fields, after the Bitvector[N] + */ +function readVariableOffsets( + data: DataView, + start: number, + end: number, + optionalFieldsEnd: number, + fixedEnd: number, + variableOffsetsPosition: number[] +): number[] { + // Since variable-sized values can be interspersed with fixed-sized values, we precalculate + // the offset indices so we can more easily deserialize the fields in once pass first we get the fixed sizes + // Note: `fixedSizes[i] = null` if that field has variable length + + const size = end - start; + const optionalFieldsByteLen = optionalFieldsEnd - start; + + // with the fixed sizes, we can read the offsets, and store for our single pass + const offsets = new Array(variableOffsetsPosition.length); + for (let i = 0; i < variableOffsetsPosition.length; i++) { + const offset = data.getUint32(start + variableOffsetsPosition[i] + optionalFieldsByteLen, true); + + // Validate offsets. If the list is empty the offset points to the end of the buffer, offset == size + if (offset > size) { + throw new Error(`Offset out of bounds ${offset} > ${size}`); + } + if (i === 0) { + if (offset !== fixedEnd) { + throw new Error(`First offset must equal to fixedEnd ${offset} != ${fixedEnd}`); + } + } else { + if (offset < offsets[i - 1]) { + throw new Error(`Offsets must be increasing ${offset} < ${offsets[i - 1]}`); + } + } + + offsets[i] = offset; + } + + return offsets; +} + +/** + * Precompute sizes of the Container doing one pass over fields + */ +function precomputeSizes(fields: Record>): { + minLen: number; + maxLen: number; + fixedSize: number | null; +} { + let minLen = 0; + let maxLen = 0; + let fixedSize: number | null = 0; + + for (const fieldType of Object.values(fields)) { + minLen += fieldType.minSize; + maxLen += fieldType.maxSize; + + if (fieldType.fixedSize === null) { + // +4 for the offset + minLen += 4; + maxLen += 4; + fixedSize = null; + } else if (fixedSize !== null) { + fixedSize += fieldType.fixedSize; + } + } + return {minLen, maxLen, fixedSize}; +} + +/** + * Compute the JSON key for each fieldName. There will exist a single JSON representation for each type. + * To transform JSON payloads to a casing that is different from the type's defined use external tooling. + */ +export function precomputeJsonKey>>( + fieldName: keyof Fields, + casingMap?: CasingMap, + jsonCase?: KeyCase +): string { + if (casingMap) { + const keyFromCaseMap = casingMap[fieldName]; + if (keyFromCaseMap === undefined) { + throw Error(`casingMap[${fieldName}] not defined`); + } + return keyFromCaseMap as string; + } else if (jsonCase) { + return Case[jsonCase](fieldName as string); + } else { + return fieldName as string; + } +} + +/** + * Render field typeNames for a detailed typeName of this Container + */ +export function renderContainerTypeName>>( + fields: Fields, + prefix = "Profile" +): string { + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + const fieldTypeNames = fieldNames.map((fieldName) => `${fieldName}: ${fields[fieldName].typeName}`).join(", "); + return `${prefix}({${fieldTypeNames}})`; +} diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts new file mode 100644 index 00000000..bf8b94fa --- /dev/null +++ b/packages/ssz/src/type/stableContainer.ts @@ -0,0 +1,829 @@ +import { + Node, + BranchNode, + LeafNode, + getNodesAtDepth, + subtreeFillToContents, + Tree, + Gindex, + toGindex, + concatGindices, + getNode, + zeroNode, + zeroHash, + countToDepth, + getNodeH, + setNode, + setNodeWithFn, +} from "@chainsafe/persistent-merkle-tree"; +import { + ValueWithCachedPermanentRoot, + hash64, + maxChunksToDepth, + merkleize, + splitIntoRootChunks, + symbolCachedPermanentRoot, +} from "../util/merkleize"; +import {Require} from "../util/types"; +import {namedClass} from "../util/named"; +import {JsonPath, Type, ValueOf} from "./abstract"; +import {CompositeType, ByteViews, CompositeTypeAny, isCompositeType} from "./composite"; +import { + getContainerTreeViewClass, + ValueOfFields, + FieldEntry, + ContainerTreeViewType, + ContainerTreeViewTypeConstructor, + computeSerdesData, +} from "../view/stableContainer"; +import { + getContainerTreeViewDUClass, + ContainerTreeViewDUType, + ContainerTreeViewDUTypeConstructor, +} from "../viewDU/stableContainer"; +import {Case} from "../util/strings"; +import {isOptionalType, toNonOptionalType, NonOptionalFields} from "./optional"; +import {BitArray} from "../value/bitArray"; +/* eslint-disable @typescript-eslint/member-ordering */ + +type BytesRange = {start: number; end: number}; + +export type StableContainerOptions> = { + typeName?: string; + jsonCase?: KeyCase; + casingMap?: CasingMap; + cachePermanentRootStruct?: boolean; + getContainerTreeViewClass?: typeof getContainerTreeViewClass; + getContainerTreeViewDUClass?: typeof getContainerTreeViewDUClass; +}; + +export type KeyCase = + | "eth2" + | "snake" + | "constant" + | "camel" + | "header" + //Same as squish + | "pascal"; + +type CasingMap> = Partial<{[K in keyof Fields]: string}>; + +/** + * StableContainer: ordered heterogeneous collection of values + * - EIP: https://eips.ethereum.org/EIPS/eip-7495 + * - Notation: Custom name per instance + */ +export class StableContainerType>> extends CompositeType< + ValueOfFields, + ContainerTreeViewType, + ContainerTreeViewDUType +> { + readonly typeName: string; + readonly depth: number; + readonly maxChunkCount: number; + readonly fixedSize: number | null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = false; + readonly isViewMutable = true; + + readonly fields: Fields; + // Precomputed data for faster serdes + readonly fieldsEntries: FieldEntry>[]; + /** End of fixed section of serialized Container */ + // readonly fixedEnd: number; + protected readonly fieldsGindex: Record; + protected readonly jsonKeyToFieldName: Record; + + /** Cached TreeView constuctor with custom prototype for this Type's properties */ + protected readonly TreeView: ContainerTreeViewTypeConstructor; + protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; + private padActiveFields: boolean[]; + + constructor(fields: Fields, readonly maxFields: number, readonly opts?: StableContainerOptions) { + super(); + + this.fields = fields; + + // Render detailed typeName. Consumers should overwrite since it can get long + this.typeName = opts?.typeName ?? renderContainerTypeName(fields); + + this.maxChunkCount = maxFields; + // Add 1 for the mixed-in bitvector + this.depth = maxChunksToDepth(this.maxChunkCount) + 1; + + // Precalculated data for faster serdes + this.fieldsEntries = []; + for (const fieldName of Object.keys(fields) as (keyof Fields)[]) { + const fieldType = fields[fieldName]; + + this.fieldsEntries.push({ + fieldName, + fieldType: toNonOptionalType(fieldType), + jsonKey: precomputeJsonKey(fieldName, opts?.casingMap, opts?.jsonCase), + gindex: toGindex(this.depth, BigInt(this.fieldsEntries.length)), + optional: isOptionalType(fieldType), + }); + } + + this.padActiveFields = Array.from({length: this.maxChunkCount - this.fieldsEntries.length}, () => false); + + if (this.fieldsEntries.length === 0) { + throw Error("StableContainer must have > 0 fields"); + } + + // Precalculate for Proofs API + this.fieldsGindex = {} as Record; + for (let i = 0; i < this.fieldsEntries.length; i++) { + this.fieldsGindex[this.fieldsEntries[i].fieldName] = toGindex(this.depth, BigInt(i)); + } + + // To resolve JSON paths in fieldName notation and jsonKey notation + this.jsonKeyToFieldName = {}; + for (const {fieldName, jsonKey} of this.fieldsEntries) { + this.jsonKeyToFieldName[jsonKey] = fieldName; + } + + const {minLen, maxLen, fixedSize} = precomputeSizes(this.fieldsEntries); + this.minSize = minLen; + this.maxSize = maxLen; + this.fixedSize = fixedSize; + + // TODO: This options are necessary for ContainerNodeStruct to override this. + // Refactor this constructor to allow customization without pollutin the options + this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); + this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + } + + static named>>( + fields: Fields, + maxFields: number, + opts: Require, "typeName"> + ): StableContainerType { + return new (namedClass(StableContainerType, opts.typeName))(fields, maxFields, opts); + } + + defaultValue(): ValueOfFields { + const value = {} as ValueOfFields; + for (const {fieldName, fieldType, optional} of this.fieldsEntries) { + value[fieldName] = (optional ? null : fieldType.defaultValue()) as ValueOf; + } + return value; + } + + getView(tree: Tree): ContainerTreeViewType { + return new this.TreeView(this, tree); + } + + getViewDU(node: Node, cache?: unknown): ContainerTreeViewDUType { + return new this.TreeViewDU(this, node, cache); + } + + cacheOfViewDU(view: ContainerTreeViewDUType): unknown { + return view.cache; + } + + commitView(view: ContainerTreeViewType): Node { + return view.node; + } + + commitViewDU(view: ContainerTreeViewDUType): Node { + view.commit(); + return view.node; + } + + // Serialization + deserialization + // ------------------------------- + // Containers can mix fixed length and variable length data. + // + // Fixed part Variable part + // [field1 offset][field2 data ][field1 data ] + // [0x000000c] [0xaabbaabbaabbaabb][0xffffffffffffffffffffffff] + + value_serializedSize(value: ValueOfFields): number { + let totalSize = Math.ceil(this.maxChunkCount / 8); + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + // skip optional fields with nullish values + if (optional && value[fieldName] == null) { + continue; + } + + // Offset (4 bytes) + size + totalSize += + fieldType.fixedSize === null ? 4 + fieldType.value_serializedSize(value[fieldName]) : fieldType.fixedSize; + } + + return totalSize; + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOfFields): number { + // compute active field bitvector + const activeFields = BitArray.fromBoolArray([ + ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), + ...this.padActiveFields, + ]); + // write active field bitvector + output.uint8Array.set(activeFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(activeFields, this.fieldsEntries); + + const activeFieldsLen = activeFields.uint8Array.length; + let fixedIndex = offset + activeFieldsLen; + let variableIndex = offset + fixedEnd; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + // skip optional fields with nullish values + if (optional && value[fieldName] == null) { + continue; + } + + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - activeFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.value_serializeToBytes(output, variableIndex, value[fieldName]); + } else { + fixedIndex = fieldType.value_serializeToBytes(output, fixedIndex, value[fieldName]); + } + } + return variableIndex; + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + const {activeFields, fieldRanges} = this.getFieldRanges(data, start, end); + const value = {} as {[K in keyof Fields]: unknown}; + + for (let i = 0, rangesIx = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && !activeFields.get(i)) { + value[fieldName] = null; + continue; + } + + const fieldRange = fieldRanges[rangesIx++]; + value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + return value as ValueOfFields; + } + + tree_serializedSize(node: Node): number { + const activeFields = this.tree_getActiveFields(node); + let totalSize = Math.ceil(activeFields.bitLen / 8); + const nodes = getNodesAtDepth(node, this.depth, 0, this.fieldsEntries.length) as Node[]; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, optional} = this.fieldsEntries[i]; + const node = nodes[i]; + if (optional && !activeFields.get(i)) { + continue; + } + + // Offset (4 bytes) + size + totalSize += fieldType.fixedSize === null ? 4 + fieldType.tree_serializedSize(node) : fieldType.fixedSize; + } + return totalSize; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + // compute active field bitvector + const activeFields = this.tree_getActiveFields(node); + // write active field bitvector + output.uint8Array.set(activeFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(activeFields, this.fieldsEntries); + + const activeFieldsLen = activeFields.uint8Array.length; + let fixedIndex = offset + activeFieldsLen; + let variableIndex = offset + fixedEnd; + + const nodes = getNodesAtDepth(node, this.depth, 0, this.fieldsEntries.length); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, optional} = this.fieldsEntries[i]; + if (optional && !activeFields.get(i)) { + continue; + } + + const node = nodes[i]; + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - activeFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + return variableIndex; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const {activeFields, fieldRanges} = this.getFieldRanges(data, start, end); + const nodes = new Array(this.fieldsEntries.length); + + for (let i = 0, rangesIx = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, optional} = this.fieldsEntries[i]; + if (optional && !activeFields.get(i)) { + nodes[i] = zeroNode(0); + continue; + } + + const fieldRange = fieldRanges[rangesIx++]; + nodes[i] = fieldType.tree_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + const rootNode = new BranchNode(subtreeFillToContents(nodes, this.depth - 1), zeroNode(0)); + return this.tree_setActiveFields(rootNode, activeFields); + } + + // Merkleization + hashTreeRoot(value: ValueOfFields): Uint8Array { + // Return cached mutable root if any + if (this.cachePermanentRootStruct) { + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + return cachedRoot; + } + } + + // compute active field bitvector + const activeFields = BitArray.fromBoolArray([ + ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), + ...this.padActiveFields, + ]); + const root = mixInActiveFields(super.hashTreeRoot(value), activeFields); + + if (this.cachePermanentRootStruct) { + (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + } + + return root; + } + + protected getRoots(struct: ValueOfFields): Uint8Array[] { + const roots = new Array(this.fieldsEntries.length); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && struct[fieldName] == null) { + roots[i] = zeroHash(0); + continue; + } + + roots[i] = fieldType.hashTreeRoot(struct[fieldName]); + } + + return roots; + } + + // Proofs + + getPropertyGindex(prop: string): Gindex | null { + const gindex = this.fieldsGindex[prop] ?? this.fieldsGindex[this.jsonKeyToFieldName[prop]]; + if (gindex === undefined) throw Error(`Unknown container property ${prop}`); + return gindex; + } + + getPropertyType(prop: string): Type { + const fieldName = this.fields[prop] ? prop : this.jsonKeyToFieldName[prop]; + const entry = this.fieldsEntries.find((entry) => entry.fieldName === fieldName); + if (entry === undefined) throw Error(`Unknown container property ${prop}`); + return entry.fieldType; + } + + getIndexProperty(index: number): string | null { + if (index >= this.fieldsEntries.length) { + return null; + } + return this.fieldsEntries[index].fieldName as string; + } + + tree_createProofGindexes(node: Node, jsonPaths: JsonPath[]): Gindex[] { + const gindexes: Gindex[] = []; + const activeFields = this.tree_getActiveFields(node); + + for (const jsonPath of jsonPaths) { + const prop = jsonPath[0]; + if (prop == null) { + continue; + } + const fieldIndex = this.fieldsEntries.findIndex((entry) => entry.fieldName === prop); + if (fieldIndex === -1) throw Error(`Unknown container property ${prop}`); + const entry = this.fieldsEntries[fieldIndex]; + if (entry.optional && !activeFields.get(fieldIndex)) { + // field is inactive and doesn't count as a leaf + continue; + } + + // same to Composite + const {type, gindex} = this.getPathInfo(jsonPath); + if (!isCompositeType(type)) { + gindexes.push(gindex); + } else { + // if the path subtype is composite, include the gindices of all the leaves + const leafGindexes = type.tree_getLeafGindices( + gindex, + type.fixedSize === null ? getNode(node, gindex) : undefined + ); + for (const gindex of leafGindexes) { + gindexes.push(gindex); + } + } + } + + return gindexes; + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const gindices: Gindex[] = []; + if (!rootNode) { + throw new Error("StableContainer.tree_getLeafGindices requires tree argument to get leaves"); + } + const activeFields = this.tree_getActiveFields(rootNode); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && !activeFields.get(i)) { + // field is inactive and doesn't count as a leaf + continue; + } + + const fieldGindex = this.fieldsGindex[fieldName]; + const fieldGindexFromRoot = concatGindices([rootGindex, fieldGindex]); + + if (fieldType.isBasic) { + gindices.push(fieldGindexFromRoot); + } else { + const compositeType = fieldType as unknown as CompositeTypeAny; + if (fieldType.fixedSize === null) { + if (!rootNode) { + throw new Error("variable type requires tree argument to get leaves"); + } + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot, getNode(rootNode, fieldGindex))); + } else { + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot)); + } + } + } + + return gindices; + } + + // JSON + + fromJson(json: unknown): ValueOfFields { + if (typeof json !== "object") { + throw Error("JSON must be of type object"); + } + if (json === null) { + throw Error("JSON must not be null"); + } + + const value = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey, optional} = this.fieldsEntries[i]; + const jsonValue = (json as Record)[jsonKey]; + if (optional && jsonValue == null) { + value[fieldName] = null as ValueOf; + continue; + } + + if (jsonValue === undefined) { + throw Error(`JSON expected key ${jsonKey} is undefined`); + } + value[fieldName] = fieldType.fromJson(jsonValue) as ValueOf; + } + + return value; + } + + toJson(value: ValueOfFields): Record { + const json: Record = {}; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey, optional} = this.fieldsEntries[i]; + if (optional && value[fieldName] == null) { + json[jsonKey] = null; + continue; + } + + json[jsonKey] = fieldType.toJson(value[fieldName]); + } + + return json; + } + + clone(value: ValueOfFields): ValueOfFields { + const newValue = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional && value[fieldName] == null) { + newValue[fieldName] = null as ValueOf; + continue; + } + + newValue[fieldName] = fieldType.clone(value[fieldName]) as ValueOf; + } + + return newValue; + } + + equals(a: ValueOfFields, b: ValueOfFields): boolean { + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, optional} = this.fieldsEntries[i]; + if (optional) { + if (a[fieldName] == null && b[fieldName] == null) { + continue; + } + if (a[fieldName] == null || b[fieldName] == null) { + return false; + } + } + if (!fieldType.equals(a[fieldName], b[fieldName])) { + return false; + } + } + + return true; + } + + /** + * `activeFields` is a bitvector prepended to the serialized data. + */ + getFieldRanges(data: ByteViews, start: number, end: number): {activeFields: BitArray; fieldRanges: BytesRange[]} { + // this.maxChunkCount = maxFields + const activeFieldsByteLen = Math.ceil(this.maxChunkCount / 8); + // active fields bitvector, do not mutate + const activeFields = new BitArray(data.uint8Array.subarray(start, start + activeFieldsByteLen), this.maxChunkCount); + + const {variableOffsetsPosition, fixedEnd, fieldRangesFixedLen, isFixedLen} = computeSerdesData( + activeFields, + this.fieldsEntries + ); + + if (variableOffsetsPosition.length === 0) { + // Validate fixed length container + const size = end - start; + if (size !== fixedEnd) { + throw Error(`${this.typeName} size ${size} not equal fixed size ${fixedEnd}`); + } + + return {activeFields, fieldRanges: fieldRangesFixedLen}; + } + + // Read offsets in one pass + const offsets = readVariableOffsets( + data.dataView, + start, + end, + activeFieldsByteLen, + fixedEnd, + variableOffsetsPosition + ); + offsets.push(end - start); // The offsets are relative to the start + + // Merge fieldRangesFixedLen + offsets in one array + let variableIdx = 0; + let fixedIdx = 0; + const fieldRanges = new Array(isFixedLen.length); + + for (let i = 0; i < isFixedLen.length; i++) { + if (isFixedLen[i]) { + // push from fixLen ranges ++ + fieldRanges[i] = fieldRangesFixedLen[fixedIdx++]; + } else { + // push from varLen ranges ++ + fieldRanges[i] = {start: offsets[variableIdx], end: offsets[variableIdx + 1]}; + variableIdx++; + } + } + return {activeFields, fieldRanges}; + } + + // helpers for the active fields + tree_getActiveFields(rootNode: Node): BitArray { + // this.maxChunkCount = maxFields + return getActiveFields(rootNode, this.maxChunkCount); + } + + tree_setActiveFields(rootNode: Node, activeFields: BitArray): Node { + return setActiveFields(rootNode, activeFields); + } + + tree_getActiveField(rootNode: Node, fieldIndex: number): boolean { + return getActiveField(rootNode, this.maxChunkCount, fieldIndex); + } + + tree_setActiveField(rootNode: Node, fieldIndex: number, value: boolean): Node { + return setActiveField(rootNode, this.maxChunkCount, fieldIndex, value); + } +} + +/** + * Returns the byte ranges of all variable size fields. + */ +function readVariableOffsets( + data: DataView, + start: number, + end: number, + activeFieldsEnd: number, + fixedEnd: number, + variableOffsetsPosition: number[] +): number[] { + // Since variable-sized values can be interspersed with fixed-sized values, we precalculate + // the offset indices so we can more easily deserialize the fields in once pass first we get the fixed sizes + // Note: `fixedSizes[i] = null` if that field has variable length + + const size = end - start; + const activeFieldsByteLen = activeFieldsEnd - start; + + // with the fixed sizes, we can read the offsets, and store for our single pass + const offsets = new Array(variableOffsetsPosition.length); + for (let i = 0; i < variableOffsetsPosition.length; i++) { + const offset = data.getUint32(start + variableOffsetsPosition[i], true) + activeFieldsByteLen; + + // Validate offsets. If the list is empty the offset points to the end of the buffer, offset == size + if (offset > size) { + throw new Error(`Offset out of bounds ${offset} > ${size}`); + } + if (i === 0) { + if (offset !== fixedEnd) { + throw new Error(`First offset must equal to fixedEnd ${offset} != ${fixedEnd}`); + } + } else { + if (offset < offsets[i - 1]) { + throw new Error(`Offsets must be increasing ${offset} < ${offsets[i - 1]}`); + } + } + + offsets[i] = offset; + } + + return offsets; +} + +/** + * Precompute sizes of the Container doing one pass over fields + */ +function precomputeSizes>>( + fields: FieldEntry[] +): { + minLen: number; + maxLen: number; + fixedSize: number | null; +} { + // at a minimum, the active fields bitvector is prepended + const activeFieldsLen = Math.ceil(fields.length / 8); + + let minLen = activeFieldsLen; + let maxLen = activeFieldsLen; + const fixedSize = null; + + for (const {fieldType, optional} of fields) { + minLen += optional ? 0 : fieldType.minSize; + maxLen += fieldType.maxSize; + + if (fieldType.fixedSize === null) { + // +4 for the offset + minLen += optional ? 0 : 4; + maxLen += 4; + } + } + return {minLen, maxLen, fixedSize}; +} + +/** + * Compute the JSON key for each fieldName. There will exist a single JSON representation for each type. + * To transform JSON payloads to a casing that is different from the type's defined use external tooling. + */ +export function precomputeJsonKey>>( + fieldName: keyof Fields, + casingMap?: CasingMap, + jsonCase?: KeyCase +): string { + if (casingMap) { + const keyFromCaseMap = casingMap[fieldName]; + if (keyFromCaseMap === undefined) { + throw Error(`casingMap[${fieldName}] not defined`); + } + return keyFromCaseMap as string; + } else if (jsonCase) { + return Case[jsonCase](fieldName as string); + } else { + return fieldName as string; + } +} + +/** + * Render field typeNames for a detailed typeName of this Container + */ +export function renderContainerTypeName>>( + fields: Fields, + prefix = "StableContainer" +): string { + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + const fieldTypeNames = fieldNames.map((fieldName) => `${fieldName}: ${fields[fieldName].typeName}`).join(", "); + return `${prefix}({${fieldTypeNames}})`; +} + +/** + * Get the active field bitvector, given the root of the tree and # of fields + */ +export function getActiveFields(rootNode: Node, bitLen: number): BitArray { + // fast path for depth 1, the bitvector fits in one chunk + if (bitLen <= 256) { + return new BitArray(rootNode.right.root.subarray(0, Math.ceil(bitLen / 8)), bitLen); + } + + const activeFieldsBuf = new Uint8Array(Math.ceil(bitLen / 8)); + const depth = countToDepth(BigInt(Math.ceil(activeFieldsBuf.length / 32))); + const nodes = getNodesAtDepth(rootNode.right, depth, 0, Math.ceil(bitLen / 256)); + for (let i = 0; i < nodes.length; i++) { + activeFieldsBuf.set(nodes[i].root, i * 32); + } + + return new BitArray(activeFieldsBuf, bitLen); +} + +export function setActiveFields(rootNode: Node, activeFields: BitArray): Node { + // fast path for depth 1, the bitvector fits in one chunk + if (activeFields.bitLen <= 256) { + const activeFieldsBuf = new Uint8Array(32); + activeFieldsBuf.set(activeFields.uint8Array); + return new BranchNode(rootNode.left, LeafNode.fromRoot(activeFieldsBuf)); + } + + const activeFieldsChunkCount = Math.ceil(activeFields.bitLen / 256); + const nodes: Node[] = []; + for (let i = 0; i < activeFieldsChunkCount; i++) { + const activeFieldsBuf = new Uint8Array(32); + activeFieldsBuf.set(activeFields.uint8Array.subarray(i * 32, (i + 1) * 32)); + nodes.push(LeafNode.fromRoot(activeFieldsBuf)); + } + + return new BranchNode(rootNode.left, subtreeFillToContents(nodes, Math.ceil(Math.log2(activeFieldsChunkCount)))); +} + +export function getActiveField(rootNode: Node, bitLen: number, fieldIndex: number): boolean { + const hIndex = Math.floor(fieldIndex / 32); + const hBitIndex = fieldIndex % 32; + + // fast path for depth 1, the bitvector fits in one chunk + if (bitLen <= 256) { + const h = getNodeH(rootNode.right, hIndex); + return Boolean(h & (1 << hBitIndex)); + } + + const chunkCount = Math.ceil(bitLen / 256); + const chunkIx = bitLen % 256; + const depth = Math.ceil(Math.log2(chunkCount)); + + const chunk = getNode(rootNode, toGindex(depth, BigInt(chunkIx))); + const h = getNodeH(chunk, hIndex); + return Boolean(h & (1 << hBitIndex)); +} + +export function setActiveField(rootNode: Node, bitLen: number, fieldIndex: number, value: boolean): Node { + const byteIx = Math.floor(fieldIndex / 8); + const bitIx = fieldIndex % 8; + + // fast path for depth 1, the bitvector fits in one chunk + if (bitLen <= 256) { + const activeFieldsBuf = rootNode.right.root; + activeFieldsBuf[byteIx] |= (value ? 1 : 0) << bitIx; + + const activeFieldGindex = BigInt(3); + return setNode(rootNode, activeFieldGindex, LeafNode.fromRoot(activeFieldsBuf)); + } + + const chunkCount = Math.ceil(bitLen / 256); + const chunkIx = bitLen % 256; + const depth = Math.ceil(Math.log2(chunkCount)); + const activeFieldsNode = rootNode.right; + const newActiveFieldsNode = setNodeWithFn(activeFieldsNode, BigInt(2 * depth + chunkIx), (node) => { + const chunkBuf = node.root; + chunkBuf[byteIx] |= (value ? 1 : 0) << bitIx; + return LeafNode.fromRoot(chunkBuf); + }); + + return new BranchNode(rootNode.left, newActiveFieldsNode); +} + +export function mixInActiveFields(root: Uint8Array, activeFields: BitArray): Uint8Array { + // fast path for depth 1, the bitvector fits in one chunk + if (activeFields.bitLen <= 256) { + const activeFieldsChunk = new Uint8Array(32); + activeFieldsChunk.set(activeFields.uint8Array); + return hash64(root, activeFieldsChunk); + } + + const activeFieldsChunks = splitIntoRootChunks(activeFields.uint8Array); + const activeFieldsRoot = merkleize(activeFieldsChunks, activeFieldsChunks.length); + return hash64(root, activeFieldsRoot); +} diff --git a/packages/ssz/src/view/container.ts b/packages/ssz/src/view/container.ts index c062e2cd..34519246 100644 --- a/packages/ssz/src/view/container.ts +++ b/packages/ssz/src/view/container.ts @@ -3,6 +3,7 @@ import {Type, ValueOf} from "../type/abstract"; import {isBasicType, BasicType} from "../type/basic"; import {isCompositeType, CompositeType} from "../type/composite"; import {TreeView} from "./abstract"; +import {NonOptionalFields} from "../type/optional"; export type FieldEntry>> = { fieldName: keyof Fields; @@ -12,13 +13,16 @@ export type FieldEntry>> = { }; /** Expected API of this View's type. This interface allows to break a recursive dependency between types and views */ -export type ContainerTypeGeneric>> = CompositeType< +export type BasicContainerTypeGeneric>> = CompositeType< ValueOfFields, ContainerTreeViewType, unknown > & { readonly fields: Fields; - readonly fieldsEntries: FieldEntry[]; + readonly fieldsEntries: (FieldEntry | FieldEntry>)[]; +}; + +export type ContainerTypeGeneric>> = BasicContainerTypeGeneric & { readonly fixedEnd: number; }; @@ -35,7 +39,7 @@ export type FieldsView>> = { }; export type ContainerTreeViewType>> = FieldsView & - TreeView>; + TreeView>; export type ContainerTreeViewTypeConstructor>> = { new (type: ContainerTypeGeneric, tree: Tree): ContainerTreeViewType; }; diff --git a/packages/ssz/src/view/profile.ts b/packages/ssz/src/view/profile.ts new file mode 100644 index 00000000..7f0bcd4e --- /dev/null +++ b/packages/ssz/src/view/profile.ts @@ -0,0 +1,219 @@ +import { + getNodeAtDepth, + Gindex, + zeroNode, + LeafNode, + Node, + toGindexBitstring, + Tree, +} from "@chainsafe/persistent-merkle-tree"; +import {Type, ValueOf} from "../type/abstract"; +import {isBasicType, BasicType} from "../type/basic"; +import {isCompositeType, CompositeType} from "../type/composite"; +import {TreeView} from "./abstract"; +import {BitArray} from "../value/bitArray"; +import {NonOptionalFields} from "../type/optional"; + +export type FieldEntry>> = { + fieldName: keyof Fields; + fieldType: Fields[keyof Fields]; + jsonKey: string; + gindex: Gindex; + // the position within the activeFields + chunkIndex: number; + optional: boolean; +}; + +/** Expected API of this View's type. This interface allows to break a recursive dependency between types and views */ +export type ContainerTypeGeneric>> = CompositeType< + ValueOfFields, + ContainerTreeViewType, + unknown +> & { + readonly fields: Fields; + readonly fieldsEntries: FieldEntry>[]; + readonly activeFields: BitArray; +}; + +export type ValueOfFields>> = {[K in keyof Fields]: ValueOf}; + +export type FieldsView>> = { + [K in keyof Fields]: Fields[K] extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TV + : // If basic, return struct value. Will NOT propagate changes upwards + Fields[K] extends BasicType + ? V + : never; +}; + +export type ContainerTreeViewType>> = FieldsView & + TreeView>; +export type ContainerTreeViewTypeConstructor>> = { + new (type: ContainerTypeGeneric, tree: Tree): ContainerTreeViewType; +}; + +/** + * Intented usage: + * + * - Get initial BeaconState from disk. + * - Before applying next block, switch to mutable + * - Get some field, create a view in mutable mode + * - Do modifications of the state in the state transition function + * - When done, commit and apply new root node once to og BeaconState + * - However, keep all the caches and transfer them to the new BeaconState + * + * Questions: + * - Can the child views created in mutable mode switch to not mutable? If so, it seems that it needs to recursively + * iterate the entire data structure and views + * + */ +class ProfileTreeView>> extends TreeView> { + constructor(readonly type: ContainerTypeGeneric, readonly tree: Tree) { + super(); + } + + get node(): Node { + return this.tree.rootNode; + } +} + +export function getProfileTreeViewClass>>( + type: ContainerTypeGeneric +): ContainerTreeViewTypeConstructor { + class CustomProfileTreeView extends ProfileTreeView {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, chunkIndex, optional} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomProfileTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomProfileTreeView) { + const leafNode = getNodeAtDepth(this.node, this.type.depth, chunkIndex) as LeafNode; + if (optional && leafNode === zeroNode(0)) { + return null; + } + + return fieldType.tree_getFromNode(leafNode); + }, + + set: function (this: CustomProfileTreeView, value) { + if (optional && value == null) { + const leafNode = zeroNode(0); + this.tree.setNodeAtDepth(this.type.depth, chunkIndex, leafNode); + return; + } + + const leafNodePrev = getNodeAtDepth(this.node, this.type.depth, chunkIndex) as LeafNode; + const leafNode = leafNodePrev.clone(); + fieldType.tree_setToNode(leafNode, value); + this.tree.setNodeAtDepth(this.type.depth, chunkIndex, leafNode); + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView. The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomProfileTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeView of fieldName + get: function (this: CustomProfileTreeView) { + const gindex = toGindexBitstring(this.type.depth, chunkIndex); + const tree = this.tree.getSubtree(gindex); + if (optional && tree.rootNode === zeroNode(0)) { + return null; + } + + return fieldType.getView(tree); + }, + + // Expects TreeView of fieldName + set: function (this: CustomProfileTreeView, value: unknown) { + if (optional && value == null) { + this.tree.setNodeAtDepth(this.type.depth, chunkIndex, zeroNode(0)); + } + + const node = fieldType.commitView(value); + this.tree.setNodeAtDepth(this.type.depth, chunkIndex, node); + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomProfileTreeView, "name", {value: type.typeName, writable: false}); + + return CustomProfileTreeView as unknown as ContainerTreeViewTypeConstructor; +} + +// TODO: deduplicate +type BytesRange = {start: number; end: number}; + +/** + * Precompute fixed and variable offsets position for faster deserialization. + * @returns Does a single pass over all fields and returns: + * - isFixedLen: If field index [i] is fixed length + * - fieldRangesFixedLen: For fields with fixed length, their range of bytes + * - variableOffsetsPosition: Position of the 4 bytes offset for variable size fields + * - fixedEnd: End of the fixed size range + * - offsets are relative to the start of serialized active fields, after the Bitvector[N] of optional fields + */ +export function computeSerdesData>>( + optionalFields: BitArray, + fields: FieldEntry[] +): { + isFixedLen: boolean[]; + fieldRangesFixedLen: BytesRange[]; + variableOffsetsPosition: number[]; + fixedEnd: number; +} { + const isFixedLen: boolean[] = []; + const fieldRangesFixedLen: BytesRange[] = []; + const variableOffsetsPosition: number[] = []; + // should not be optionalFields.uint8Array.length because offsets are relative to the start of serialized active fields + let pointerFixed = 0; + + let optionalIndex = 0; + for (const {optional, fieldType} of fields) { + if (optional) { + if (!optionalFields.get(optionalIndex++)) { + continue; + } + } + + isFixedLen.push(fieldType.fixedSize !== null); + if (fieldType.fixedSize === null) { + // Variable length + variableOffsetsPosition.push(pointerFixed); + pointerFixed += 4; + } else { + fieldRangesFixedLen.push({start: pointerFixed, end: pointerFixed + fieldType.fixedSize}); + pointerFixed += fieldType.fixedSize; + } + } + + return { + isFixedLen, + fieldRangesFixedLen, + variableOffsetsPosition, + fixedEnd: pointerFixed, + }; +} diff --git a/packages/ssz/src/view/stableContainer.ts b/packages/ssz/src/view/stableContainer.ts new file mode 100644 index 00000000..f7619619 --- /dev/null +++ b/packages/ssz/src/view/stableContainer.ts @@ -0,0 +1,247 @@ +import { + getNodeAtDepth, + Gindex, + LeafNode, + Node, + toGindexBitstring, + Tree, + zeroNode, +} from "@chainsafe/persistent-merkle-tree"; +import {Type, ValueOf} from "../type/abstract"; +import {isBasicType, BasicType} from "../type/basic"; +import {isCompositeType, CompositeType} from "../type/composite"; +import {TreeView} from "./abstract"; +import {NonOptionalFields, OptionalType} from "../type/optional"; +import {BitArray} from "../value/bitArray"; + +// some code is here to break the circular dependency between type, view, and viewDU + +export type FieldEntry>> = { + fieldName: keyof Fields; + fieldType: Fields[keyof Fields]; + jsonKey: string; + gindex: Gindex; + optional: boolean; +}; + +/** Expected API of this View's type. This interface allows to break a recursive dependency between types and views */ +export type StableContainerTypeGeneric>> = CompositeType< + ValueOfFields, + ContainerTreeViewType, + unknown +> & { + readonly fields: Fields; + readonly fieldsEntries: FieldEntry>[]; + + tree_getActiveFields: (node: Node) => BitArray; + tree_setActiveFields: (node: Node, activeFields: BitArray) => Node; + tree_getActiveField: (node: Node, fieldIndex: number) => boolean; + tree_setActiveField: (node: Node, fieldIndex: number, value: boolean) => Node; +}; + +export type ValueOfFields>> = {[K in keyof Fields]: ValueOf}; + +export type ViewType> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TV + : // If basic, return struct value. Will NOT propagate changes upwards + T extends BasicType + ? V + : never; + +export type OptionalViewType> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards if not nullish + TV | null | undefined + : // If basic, return struct value or nullish. Will NOT propagate changes upwards + T extends BasicType + ? V | null | undefined + : never; + +export type FieldsView>> = { + [K in keyof Fields]: Fields[K] extends OptionalType ? OptionalViewType : ViewType; +}; + +export type ContainerTreeViewType>> = FieldsView & + TreeView>; +export type ContainerTreeViewTypeConstructor>> = { + new (type: StableContainerTypeGeneric, tree: Tree): ContainerTreeViewType; +}; + +/** + * Intented usage: + * + * - Get initial BeaconState from disk. + * - Before applying next block, switch to mutable + * - Get some field, create a view in mutable mode + * - Do modifications of the state in the state transition function + * - When done, commit and apply new root node once to og BeaconState + * - However, keep all the caches and transfer them to the new BeaconState + * + * Questions: + * - Can the child views created in mutable mode switch to not mutable? If so, it seems that it needs to recursively + * iterate the entire data structure and views + * + */ +class ContainerTreeView>> extends TreeView< + StableContainerTypeGeneric +> { + constructor(readonly type: StableContainerTypeGeneric, readonly tree: Tree) { + super(); + } + + get node(): Node { + return this.tree.rootNode; + } +} + +export function getContainerTreeViewClass>>( + type: StableContainerTypeGeneric +): ContainerTreeViewTypeConstructor { + class CustomContainerTreeView extends ContainerTreeView {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, optional} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomContainerTreeView) { + const leafNode = getNodeAtDepth(this.node, this.type.depth, index) as LeafNode; + if (optional && this.type.tree_getActiveField(this.tree.rootNode, index) === false) { + return null; + } + return fieldType.tree_getFromNode(leafNode); + }, + + set: function (this: CustomContainerTreeView, value) { + if (optional && value == null) { + this.tree.setNodeAtDepth(this.type.depth, index, zeroNode(0)); + // only update the active field if necessary + if (this.type.tree_getActiveField(this.tree.rootNode, index)) { + this.tree.rootNode = this.type.tree_setActiveField(this.tree.rootNode, index, false); + } + return; + } + const leafNodePrev = getNodeAtDepth(this.node, this.type.depth, index) as LeafNode; + const leafNode = leafNodePrev.clone(); + fieldType.tree_setToNode(leafNode, value); + this.tree.setNodeAtDepth(this.type.depth, index, leafNode); + // only update the active field if necessary + if (!this.type.tree_getActiveField(this.tree.rootNode, index)) { + this.tree.rootNode = this.type.tree_setActiveField(this.tree.rootNode, index, true); + } + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView (if not nullish). The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeView of fieldName + get: function (this: CustomContainerTreeView) { + const gindex = toGindexBitstring(this.type.depth, index); + const subtree = this.tree.getSubtree(gindex); + if (optional && this.type.tree_getActiveField(this.tree.rootNode, index) === false) { + return null; + } + return fieldType.getView(subtree); + }, + + // Expects TreeView of fieldName + set: function (this: CustomContainerTreeView, value: unknown) { + if (optional && value == null) { + this.tree.setNodeAtDepth(this.type.depth, index, zeroNode(0)); + // only update the active field if necessary + if (this.type.tree_getActiveField(this.tree.rootNode, index)) { + this.tree.rootNode = this.type.tree_setActiveField(this.tree.rootNode, index, false); + } + return; + } + const node = fieldType.commitView(value); + this.tree.setNodeAtDepth(this.type.depth, index, node); + // only update the active field if necessary + if (!this.type.tree_getActiveField(this.tree.rootNode, index)) { + this.tree.rootNode = this.type.tree_setActiveField(this.tree.rootNode, index, false); + } + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomContainerTreeView, "name", {value: type.typeName, writable: false}); + + return CustomContainerTreeView as unknown as ContainerTreeViewTypeConstructor; +} + +type BytesRange = {start: number; end: number}; + +/** + * Precompute fixed and variable offsets position for faster deserialization. + * @throws when activeFields does not align with non-optional field types + * @returns Does a single pass over all fields and returns: + * - isFixedLen: If field index [i] is fixed length + * - fieldRangesFixedLen: For fields with fixed length, their range of bytes + * - variableOffsetsPosition: Position of the 4 bytes offset for variable size fields + * - fixedEnd: End of the fixed size range + * - + */ +export function computeSerdesData>>( + activeFields: BitArray, + fields: FieldEntry[] +): { + isFixedLen: boolean[]; + fieldRangesFixedLen: BytesRange[]; + variableOffsetsPosition: number[]; + fixedEnd: number; +} { + const isFixedLen: boolean[] = []; + const fieldRangesFixedLen: BytesRange[] = []; + const variableOffsetsPosition: number[] = []; + let pointerFixed = Math.ceil(activeFields.bitLen / 8); + + for (const [i, {fieldName, fieldType, optional}] of fields.entries()) { + // if the field is inactive + if (!activeFields.get(i)) { + if (!optional) { + throw new Error(`Field "${String(fieldName)}" must be active since it is not optional`); + } + continue; + } + + isFixedLen.push(fieldType.fixedSize !== null); + if (fieldType.fixedSize === null) { + // Variable length + variableOffsetsPosition.push(pointerFixed); + pointerFixed += 4; + } else { + fieldRangesFixedLen.push({start: pointerFixed, end: pointerFixed + fieldType.fixedSize}); + pointerFixed += fieldType.fixedSize; + } + } + + return { + isFixedLen, + fieldRangesFixedLen, + variableOffsetsPosition, + fixedEnd: pointerFixed, + }; +} diff --git a/packages/ssz/src/viewDU/container.ts b/packages/ssz/src/viewDU/container.ts index 3e037e0a..993ff602 100644 --- a/packages/ssz/src/viewDU/container.ts +++ b/packages/ssz/src/viewDU/container.ts @@ -9,7 +9,7 @@ import { import {ByteViews, Type} from "../type/abstract"; import {BasicType, isBasicType} from "../type/basic"; import {CompositeType, isCompositeType, CompositeTypeAny} from "../type/composite"; -import {ContainerTypeGeneric} from "../view/container"; +import {BasicContainerTypeGeneric, ContainerTypeGeneric} from "../view/container"; import {TreeViewDU} from "./abstract"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -30,14 +30,16 @@ export type ContainerTreeViewDUTypeConstructor, node: Node, cache?: unknown): ContainerTreeViewDUType; }; +export type ChangedNode = {index: number; node: Node}; + type ContainerTreeViewDUCache = { nodes: Node[]; caches: unknown[]; nodesPopulated: boolean; }; -class ContainerTreeViewDU>> extends TreeViewDU< - ContainerTypeGeneric +export class BasicContainerTreeViewDU>> extends TreeViewDU< + BasicContainerTypeGeneric > { protected nodes: Node[] = []; protected caches: unknown[]; @@ -46,7 +48,7 @@ class ContainerTreeViewDU>> extends private nodesPopulated: boolean; constructor( - readonly type: ContainerTypeGeneric, + readonly type: BasicContainerTypeGeneric, protected _rootNode: Node, cache?: ContainerTreeViewDUCache ) { @@ -94,7 +96,7 @@ class ContainerTreeViewDU>> extends // if old root is not hashed, no need to pass hcByLevel to child view bc we need to do full traversal here const byLevelView = hcByLevel != null && isOldRootHashed ? hcByLevel : null; - const nodesChanged: {index: number; node: Node}[] = []; + const nodesChanged: ChangedNode[] = []; for (const [index, view] of this.viewsChanged) { const fieldType = this.type.fieldsEntries[index].fieldType as unknown as CompositeTypeAny; @@ -114,8 +116,7 @@ class ContainerTreeViewDU>> extends // TODO: Optimize to loop only once, Numerical sort ascending const nodesChangedSorted = nodesChanged.sort((a, b) => a.index - b.index); - const indexes = nodesChangedSorted.map((entry) => entry.index); - const nodes = nodesChangedSorted.map((entry) => entry.node); + const {indexes, nodes} = this.parseNodesChanged(nodesChangedSorted); this._rootNode = setNodesAtDepth( this._rootNode, @@ -135,6 +136,12 @@ class ContainerTreeViewDU>> extends this.viewsChanged.clear(); } + protected parseNodesChanged(nodes: ChangedNode[]): {indexes: number[]; nodes: Node[]} { + const indexes = nodes.map((entry) => entry.index); + const nodesArray = nodes.map((entry) => entry.node); + return {indexes, nodes: nodesArray}; + } + protected clearCache(): void { this.nodes = []; this.caches = []; @@ -147,6 +154,16 @@ class ContainerTreeViewDU>> extends // However preserving _SOME_ caches results in a very unpredictable experience. this.viewsChanged.clear(); } +} + +class ContainerTreeViewDU>> extends BasicContainerTreeViewDU { + constructor( + readonly type: ContainerTypeGeneric, + protected _rootNode: Node, + cache?: ContainerTreeViewDUCache + ) { + super(type, _rootNode, cache); + } /** * Same method to `type/container.ts` that call ViewDU.serializeToBytes() of internal fields. diff --git a/packages/ssz/src/viewDU/profile.ts b/packages/ssz/src/viewDU/profile.ts new file mode 100644 index 00000000..3a7eaa9d --- /dev/null +++ b/packages/ssz/src/viewDU/profile.ts @@ -0,0 +1,252 @@ +import {getNodeAtDepth, LeafNode, Node, zeroNode} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews, Type} from "../type/abstract"; +import {BasicType, isBasicType} from "../type/basic"; +import {CompositeType, isCompositeType} from "../type/composite"; +import {computeSerdesData, ContainerTypeGeneric} from "../view/profile"; +import {TreeViewDU} from "./abstract"; +import {BasicContainerTreeViewDU, ChangedNode} from "./container"; +import {OptionalType} from "../type/optional"; +import {BitArray} from "../value/bitArray"; + +/* eslint-disable @typescript-eslint/member-ordering */ + +export type ViewDUValue> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TVDU + : // If basic, return struct value. Will NOT propagate changes upwards + T extends BasicType + ? V + : never; + +export type OptionalViewDUValue> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TVDU | null | undefined + : // If basic, return struct value. Will NOT propagate changes upwards + T extends BasicType + ? V | null | undefined + : never; + +export type FieldsViewDU>> = { + [K in keyof Fields]: Fields[K] extends OptionalType ? OptionalViewDUValue : ViewDUValue; +}; + +export type ContainerTreeViewDUType>> = FieldsViewDU & + TreeViewDU>; +export type ContainerTreeViewDUTypeConstructor>> = { + new (type: ContainerTypeGeneric, node: Node, cache?: unknown): ContainerTreeViewDUType; +}; + +type ContainerTreeViewDUCache = { + nodes: Node[]; + caches: unknown[]; + nodesPopulated: boolean; +}; + +class ProfileTreeViewDU>> extends BasicContainerTreeViewDU { + constructor( + readonly type: ContainerTypeGeneric, + protected _rootNode: Node, + cache?: ContainerTreeViewDUCache + ) { + super(type, _rootNode, cache); + } + + protected parseNodesChanged(nodesArray: ChangedNode[]): {indexes: number[]; nodes: Node[]} { + const indexes = new Array(nodesArray.length); + const nodes = new Array(nodesArray.length); + for (const [i, change] of nodesArray.entries()) { + const {index, node} = change; + const chunkIndex = this.type.fieldsEntries[index].chunkIndex; + indexes[i] = chunkIndex; + nodes[i] = node; + } + return {indexes, nodes}; + } + + /** + * Same method to `type/profile.ts` that call ViewDU.serializeToBytes() of internal fields. + */ + serializeToBytes(output: ByteViews, offset: number): number { + this.commit(); + + const optionalArr: boolean[] = []; + for (let index = 0; index < this.type.fieldsEntries.length; index++) { + const {chunkIndex, optional} = this.type.fieldsEntries[index]; + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex); + this.nodes[index] = node; + } + if (optional) { + optionalArr.push(node !== zeroNode(0)); + } + } + + const optionalFields = BitArray.fromBoolArray(optionalArr); + output.uint8Array.set(optionalFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(optionalFields, this.type.fieldsEntries); + + const optionalFieldsLen = optionalFields.uint8Array.length; + let fixedIndex = offset + optionalFieldsLen; + let variableIndex = offset + fixedEnd + optionalFieldsLen; + for (let index = 0; index < this.type.fieldsEntries.length; index++) { + const {fieldType, optional} = this.type.fieldsEntries[index]; + const node = this.nodes[index]; + // all nodes are populated above + if (optional && node === zeroNode(0)) { + continue; + } + + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - optionalFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + // basic types always have fixedSize + if (isCompositeType(fieldType)) { + const view = fieldType.getViewDU(node, this.caches[index]) as TreeViewDU; + if (view.serializeToBytes !== undefined) { + variableIndex = view.serializeToBytes(output, variableIndex); + } else { + // some types don't define ViewDU as TreeViewDU, like the UnionType, in that case view.serializeToBytes = undefined + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } + } + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + + return variableIndex; + } +} + +export function getProfileTreeViewDUClass>>( + type: ContainerTypeGeneric +): ContainerTreeViewDUTypeConstructor { + class CustomProfileTreeViewDU extends ProfileTreeViewDU {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, chunkIndex, optional} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomProfileTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomProfileTreeViewDU) { + // First walk through the tree to get the root node for that index + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex); + this.nodes[index] = node; + } + + if (optional && node === zeroNode(0)) { + return null; + } + + return fieldType.tree_getFromNode(node as LeafNode) as unknown; + }, + + set: function (this: CustomProfileTreeViewDU, value) { + if (optional && value == null) { + this.nodes[index] = zeroNode(0); + this.nodesChanged.add(index); + return; + } + + // Create new node if current leafNode is not dirty + let nodeChanged: LeafNode; + if (this.nodesChanged.has(index)) { + // TODO: This assumes that node has already been populated + nodeChanged = this.nodes[index] as LeafNode; + } else { + const nodePrev = (this.nodes[index] ?? + getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex)) as LeafNode; + + nodeChanged = nodePrev.clone(); + // Store the changed node in the nodes cache + this.nodes[index] = nodeChanged; + this.nodesChanged.add(index); + } + + fieldType.tree_setToNode(nodeChanged, value); + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView. The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomProfileTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeViewDU of fieldName + get: function (this: CustomProfileTreeViewDU) { + const viewChanged = this.viewsChanged.get(index); + if (viewChanged) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return viewChanged; + } + + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex); + this.nodes[index] = node; + } + + if (optional && node === zeroNode(0)) { + return null; + } + + // Keep a reference to the new view to call .commit on it latter, only if mutable + const view = fieldType.getViewDU(node, this.caches[index]); + if (fieldType.isViewMutable) { + this.viewsChanged.set(index, view); + } + + // No need to persist the child's view cache since a second get returns this view instance. + // The cache is only persisted on commit where the viewsChanged map is dropped. + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return view; + }, + + // Expects TreeViewDU of fieldName + set: function (this: CustomProfileTreeViewDU, view: unknown) { + if (optional && view == null) { + this.nodes[index] = zeroNode(0); + this.nodesChanged.add(index); + return; + } + + // When setting a view: + // - Not necessary to commit node + // - Not necessary to persist cache + // Just keeping a reference to the view in this.viewsChanged ensures consistency + this.viewsChanged.set(index, view); + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomProfileTreeViewDU, "name", {value: type.typeName, writable: false}); + + return CustomProfileTreeViewDU as unknown as ContainerTreeViewDUTypeConstructor; +} diff --git a/packages/ssz/src/viewDU/stableContainer.ts b/packages/ssz/src/viewDU/stableContainer.ts new file mode 100644 index 00000000..dc892ba4 --- /dev/null +++ b/packages/ssz/src/viewDU/stableContainer.ts @@ -0,0 +1,257 @@ +import {getNodeAtDepth, LeafNode, Node, zeroNode, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews, Type} from "../type/abstract"; +import {BasicType, isBasicType} from "../type/basic"; +import {CompositeType, isCompositeType} from "../type/composite"; +import {computeSerdesData, StableContainerTypeGeneric} from "../view/stableContainer"; +import {TreeViewDU} from "./abstract"; +import {OptionalType} from "../type/optional"; +import {BitArray} from "../value/bitArray"; +import {BasicContainerTreeViewDU} from "./container"; + +/* eslint-disable @typescript-eslint/member-ordering */ + +export type ViewDUValue> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TVDU + : // If basic, return struct value. Will NOT propagate changes upwards + T extends BasicType + ? V + : never; + +export type OptionalViewDUValue> = T extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TVDU | null | undefined + : // If basic, return struct value. Will NOT propagate changes upwards + T extends BasicType + ? V | null | undefined + : never; + +export type FieldsViewDU>> = { + [K in keyof Fields]: Fields[K] extends OptionalType ? OptionalViewDUValue : ViewDUValue; +}; + +export type ContainerTreeViewDUType>> = FieldsViewDU & + TreeViewDU>; +export type ContainerTreeViewDUTypeConstructor>> = { + new (type: StableContainerTypeGeneric, node: Node, cache?: unknown): ContainerTreeViewDUType; +}; + +type ContainerTreeViewDUCache = { + activeFields: BitArray; + nodes: Node[]; + caches: unknown[]; + nodesPopulated: boolean; +}; + +class StableContainerTreeViewDU>> extends BasicContainerTreeViewDU { + /** pending active fields bitvector */ + protected activeFields: BitArray; + + constructor( + readonly type: StableContainerTypeGeneric, + protected _rootNode: Node, + cache?: ContainerTreeViewDUCache + ) { + super(type, _rootNode, cache); + + if (cache) { + this.activeFields = cache.activeFields; + } else { + this.activeFields = type.tree_getActiveFields(_rootNode); + } + } + + get cache(): ContainerTreeViewDUCache { + const result = super.cache; + return {...result, activeFields: this.activeFields}; + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + super.commit(hcOffset, hcByLevel); + this._rootNode = this.type.tree_setActiveFields(this._rootNode, this.activeFields); + if (hcByLevel !== null) { + hcByLevel[hcOffset].push(this._rootNode.left, this._rootNode.right, this._rootNode); + } + } + + /** + * Same method to `type/container.ts` that call ViewDU.serializeToBytes() of internal fields. + */ + serializeToBytes(output: ByteViews, offset: number): number { + this.commit(); + + const activeFields = this.type.tree_getActiveFields(this.node); + // write active fields bitvector + output.uint8Array.set(activeFields.uint8Array, offset); + + const {fixedEnd} = computeSerdesData(activeFields, this.type.fieldsEntries); + + const activeFieldsLen = activeFields.uint8Array.length; + let fixedIndex = offset + activeFieldsLen; + let variableIndex = offset + fixedEnd; + for (let index = 0; index < this.type.fieldsEntries.length; index++) { + const {fieldType, optional} = this.type.fieldsEntries[index]; + if (optional && !activeFields.get(index)) { + continue; + } + + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, index); + this.nodes[index] = node; + } + if (fieldType.fixedSize === null) { + // write offset relative to the start of serialized active fields, after the Bitvector[N] + output.dataView.setUint32(fixedIndex, variableIndex - offset - activeFieldsLen, true); + fixedIndex += 4; + // write serialized element to variable section + // basic types always have fixedSize + if (isCompositeType(fieldType)) { + const view = fieldType.getViewDU(node, this.caches[index]) as TreeViewDU; + if (view.serializeToBytes !== undefined) { + variableIndex = view.serializeToBytes(output, variableIndex); + } else { + // some types don't define ViewDU as TreeViewDU, like the UnionType, in that case view.serializeToBytes = undefined + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } + } + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + + return variableIndex; + } +} + +export function getContainerTreeViewDUClass>>( + type: StableContainerTypeGeneric +): ContainerTreeViewDUTypeConstructor { + class CustomContainerTreeViewDU extends StableContainerTreeViewDU {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, optional} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomContainerTreeViewDU) { + if (optional && this.activeFields.get(index) === false) { + return null; + } + + // First walk through the tree to get the root node for that index + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, index); + this.nodes[index] = node; + } + + return fieldType.tree_getFromNode(node as LeafNode) as unknown; + }, + + set: function (this: CustomContainerTreeViewDU, value) { + if (optional && value == null) { + this.nodes[index] = zeroNode(0); + this.nodesChanged.add(index); + this.activeFields.set(index, false); + return; + } + + // Create new node if current leafNode is not dirty + let nodeChanged: LeafNode; + if (this.nodesChanged.has(index)) { + // TODO: This assumes that node has already been populated + nodeChanged = this.nodes[index] as LeafNode; + } else { + const nodePrev = (this.nodes[index] ?? getNodeAtDepth(this._rootNode, this.type.depth, index)) as LeafNode; + + nodeChanged = nodePrev.clone(); + // Store the changed node in the nodes cache + this.nodes[index] = nodeChanged; + this.nodesChanged.add(index); + } + + fieldType.tree_setToNode(nodeChanged, value); + this.activeFields.set(index, true); + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView. The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeViewDU of fieldName + get: function (this: CustomContainerTreeViewDU) { + if (optional && this.activeFields.get(index) === false) { + return null; + } + + const viewChanged = this.viewsChanged.get(index); + if (viewChanged) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return viewChanged; + } + + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, index); + this.nodes[index] = node; + } + + // Keep a reference to the new view to call .commit on it latter, only if mutable + const view = fieldType.getViewDU(node, this.caches[index]); + if (fieldType.isViewMutable) { + this.viewsChanged.set(index, view); + } + + // No need to persist the child's view cache since a second get returns this view instance. + // The cache is only persisted on commit where the viewsChanged map is dropped. + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return view; + }, + + // Expects TreeViewDU of fieldName + set: function (this: CustomContainerTreeViewDU, view: unknown) { + if (optional && view == null) { + this.nodes[index] = zeroNode(0); + this.nodesChanged.add(index); + this.activeFields.set(index, false); + return; + } + + // When setting a view: + // - Not necessary to commit node + // - Not necessary to persist cache + // Just keeping a reference to the view in this.viewsChanged ensures consistency + this.viewsChanged.set(index, view); + this.activeFields.set(index, true); + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomContainerTreeViewDU, "name", {value: type.typeName, writable: false}); + + return CustomContainerTreeViewDU as unknown as ContainerTreeViewDUTypeConstructor; +} diff --git a/packages/ssz/test/unit/byType/profile/tree.test.ts b/packages/ssz/test/unit/byType/profile/tree.test.ts new file mode 100644 index 00000000..a29662af --- /dev/null +++ b/packages/ssz/test/unit/byType/profile/tree.test.ts @@ -0,0 +1,743 @@ +import {expect} from "chai"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; +import { + BitArray, + BitListType, + BitVectorType, + BooleanType, + ByteListType, + ByteVectorType, + ContainerNodeStructType, + ContainerType, + ListBasicType, + ListCompositeType, + NoneType, + OptionalType, + ProfileType, + StableContainerType, + toHexString, + UnionType, + ValueOf, + VectorBasicType, + VectorCompositeType, +} from "../../../../src"; +import {uint64NumInfType, uint64NumType} from "../../../utils/primitiveTypes"; +import {runViewTestMutation} from "../runViewTestMutation"; + +// Test both ContainerType, ContainerNodeStructType only if +// - All fields are immutable + +// TODO: different activeFields + +runViewTestMutation({ + // Use Number64UintType and NumberUintType to test they work the same + type: new ProfileType({a: uint64NumInfType, b: uint64NumInfType}, BitArray.fromBoolArray([false, true, true, false])), + mutations: [ + { + id: "set basic", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 2}, + fn: (tv) => { + tv.a = 10; + }, + }, + { + id: "set basic x2", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 20}, + fn: (tv) => { + tv.a = 10; + tv.b = 20; + }, + }, + // Test that reading a uin64 value that spans two hashObject h values works + // the same with Number64UintType and NumberUintType + { + id: "swap props", + valueBefore: {a: 0xffffffff + 1, b: 0xffffffff + 2}, + valueAfter: {a: 0xffffffff + 2, b: 0xffffffff + 1}, + fn: (tv) => { + const a = tv.a; + const b = tv.b; + tv.a = b; + tv.b = a; + }, + }, + ], +}); + +const containerUintsType = new ProfileType( + {a: uint64NumInfType, b: uint64NumInfType}, + BitArray.fromBoolArray([false, true, true, false]) +); + +runViewTestMutation({ + type: containerUintsType, + treeViewToStruct: (tv) => ({a: tv.a, b: tv.b}), + mutations: [ + { + id: "set all properties", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 20}, + fn: (tv) => { + tv.a = 10; + tv.b = 21; + // Change twice on purpose to trigger a branch in set basic + tv.b = 20; + }, + }, + ], +}); + +const byte32 = new ByteVectorType(32); +const containerBytesType = new ProfileType({a: byte32, b: byte32}, BitArray.fromBoolArray([false, true, true, false])); +const rootOf = (i: number): Buffer => Buffer.alloc(32, i); + +runViewTestMutation({ + type: containerBytesType, + treeViewToStruct: (tv) => ({a: tv.a, b: tv.b}), + mutations: [ + { + id: "set all properties", + valueBefore: {a: rootOf(1), b: rootOf(2)}, + valueAfter: {a: rootOf(3), b: rootOf(4)}, + fn: (tv) => { + tv.a = rootOf(3); + tv.b = rootOf(4); + }, + }, + ], +}); + +const profileUint64 = new ProfileType({a: uint64NumType}, BitArray.fromBoolArray([false, true, false, false])); + +describe(`${profileUint64.typeName} drop caches`, () => { + it("Make some changes then get previous value", () => { + const view = profileUint64.defaultViewDU(); + const bytesBefore = toHexString(view.serialize()); + + // Make changes to view and clone them to new view + view.a = 1; + view.clone(); + + const bytesAfter = toHexString(view.serialize()); + expect(bytesAfter).to.equal(bytesBefore, "view retained changes"); + }); +}); + +// Test only ContainerType if +// - Some fields are mutable + +const list8Uint64NumInfType = new ListBasicType(uint64NumInfType, 8); + +runViewTestMutation({ + type: new ProfileType( + {a: uint64NumInfType, b: uint64NumInfType, list: list8Uint64NumInfType}, + BitArray.fromBoolArray([false, true, true, true]) + ), + mutations: [ + { + id: "set composite entire list", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + tv.list = list8Uint64NumInfType.toViewDU([10, 20]); + }, + }, + { + id: "set composite list with push", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + tv.list.push(10); + tv.list.push(20); + }, + }, + // Test that keeping a reference to `list` and pushing twice mutates the original tv value + { + id: "set composite list with push and reference", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + const list = tv.list; + list.push(10); + list.push(20); + }, + }, + ], +}); + +const containerUint64 = new ProfileType({a: uint64NumType}, BitArray.fromBoolArray([false, true, false, false])); +const listOfContainers = new ListCompositeType(containerUint64, 4); + +runViewTestMutation({ + // Ensure mutations of child array are commited + type: new ContainerType({list: listOfContainers}), + treeViewToStruct: (tv) => { + const listArr: ValueOf = []; + for (let i = 0; i < tv.list.length; i++) { + const item = tv.list.get(i); + listArr.push({a: item.a}); + } + return {list: listArr}; + }, + mutations: [ + { + id: "Push two values", + valueBefore: {list: []}, + valueAfter: {list: [{a: 1}, {a: 2}]}, + fn: (tv) => { + tv.list.push(containerUint64.toViewDU({a: 1})); + tv.list.push(containerUint64.toViewDU({a: 2})); + }, + }, + ], +}); + +// to test new the VietDU.serialize() implementation for different types +const mixedContainer = new ProfileType( + { + // a basic type + a: uint64NumType, + // a list basic type + b: new ListBasicType(uint64NumType, 10), + // a list composite type + c: new ListCompositeType(new ContainerType({a: uint64NumInfType, b: uint64NumInfType}), 10), + // embedded container type + d: new ContainerType({a: uint64NumInfType}), + // a union type, cannot mutate through this test + e: new UnionType([new NoneType(), uint64NumInfType]), + }, + BitArray.fromBoolArray([false, true, true, false, true, true, true, false]) +); + +runViewTestMutation({ + type: mixedContainer, + mutations: [ + { + id: "increase by 1", + valueBefore: {a: 10, b: [0, 1], c: [{a: 100, b: 101}], d: {a: 1000}, e: {selector: 1, value: 2000}}, + // View/ViewDU of Union is a value so we cannot mutate + valueAfter: {a: 11, b: [1, 2], c: [{a: 101, b: 102}], d: {a: 1001}, e: {selector: 1, value: 2000}}, + fn: (tv) => { + tv.a += 1; + const b = tv.b; + for (let i = 0; i < b.length; i++) { + b.set(i, b.get(i) + 1); + } + const c = tv.c; + for (let i = 0; i < c.length; i++) { + const item = c.get(i); + item.a += 1; + item.b += 1; + } + tv.d.a += 1; + // does not affect anyway, leaving here to make it explicit + tv.e = {selector: 1, value: tv.e.value ?? 0 + 1}; + }, + }, + ], +}); + +describe("ProfileViewDU batchHashTreeRoot", function () { + const childContainerType = new ContainerType({f0: uint64NumInfType, f1: uint64NumInfType}); + const unionType = new UnionType([new NoneType(), uint64NumType]); + const listBasicType = new ListBasicType(uint64NumType, 10); + const vectorBasicType = new VectorBasicType(uint64NumType, 2); + const listCompositeType = new ListCompositeType(childContainerType, 10); + const vectorCompositeType = new VectorCompositeType(childContainerType, 1); + const bitVectorType = new BitVectorType(64); + const bitListType = new BitListType(4); + const childContainerStruct = new ContainerNodeStructType({g0: uint64NumInfType, g1: uint64NumInfType}); + const optionalType = new OptionalType(listBasicType); + const parentContainerType = new ProfileType( + { + a: uint64NumType, + b: new BooleanType(), + c: unionType, + d: new ByteListType(64), + e: new ByteVectorType(64), + // a child container type + f: childContainerType, + g: childContainerStruct, + h: listBasicType, + i: vectorBasicType, + j: listCompositeType, + k: vectorCompositeType, + l: bitVectorType, + m: bitListType, + n: optionalType, + }, + BitArray.fromBoolArray([ + false, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + ]) + ); + + const value: ValueOf = { + a: 10, + b: true, + c: {selector: 1, value: 100}, + d: Buffer.alloc(64, 2), + e: Buffer.alloc(64, 1), + f: {f0: 100, f1: 101}, + g: {g0: 100, g1: 101}, + h: [1, 2], + i: [1, 2], + j: [{f0: 1, f1: 2}], + k: [{f0: 1, f1: 2}], + l: BitArray.fromSingleBit(64, 5), + m: BitArray.fromSingleBit(4, 1), + n: [1, 2], + }; + const expectedRoot = parentContainerType.hashTreeRoot(value); + + it("fresh ViewDU", () => { + expect(parentContainerType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify Number type", () => { + const viewDU = parentContainerType.toViewDU({...value, a: 9}); + viewDU.batchHashTreeRoot(); + viewDU.a += 1; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.a = 10; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BooleanType", () => { + const viewDU = parentContainerType.toViewDU({...value, b: false}); + viewDU.batchHashTreeRoot(); + viewDU.b = true; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.b = true; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify UnionType", () => { + const viewDU = parentContainerType.toViewDU({...value, c: {selector: 1, value: 101}}); + viewDU.batchHashTreeRoot(); + viewDU.c = unionType.toViewDU({selector: 1, value: 100}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.c = unionType.toViewDU({selector: 1, value: 100}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ByteVectorType", () => { + const viewDU = parentContainerType.toViewDU(value); + viewDU.batchHashTreeRoot(); + // this takes more than 1 chunk so the resulting node is a branch node + viewDU.e = viewDU.e.slice(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.e = viewDU.e.slice(); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ByteListType", () => { + const viewDU = parentContainerType.toViewDU(value); + viewDU.batchHashTreeRoot(); + // this takes more than 1 chunk so the resulting node is a branch node + viewDU.d = viewDU.d.slice(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.d = viewDU.d.slice(); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify full child container", () => { + const viewDU = parentContainerType.toViewDU({...value, f: {f0: 99, f1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.f = childContainerType.toViewDU({f0: 100, f1: 101}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.f = childContainerType.toViewDU({f0: 100, f1: 101}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify partial child container", () => { + const viewDU = parentContainerType.toViewDU({...value, f: {f0: 99, f1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.f.f0 = 100; + viewDU.f.f1 = 101; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.f.f0 = 100; + viewDU.f.f1 = 101; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ContainerNodeStructType", () => { + const viewDU = parentContainerType.toViewDU({...value, g: {g0: 99, g1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.g = childContainerStruct.toViewDU({g0: 100, g1: 101}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.g = childContainerStruct.toViewDU({g0: 100, g1: 101}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify partial ContainerNodeStructType", () => { + const viewDU = parentContainerType.toViewDU({...value, g: {g0: 99, g1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.g.g0 = 100; + viewDU.g.g1 = 101; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.g.g0 = 100; + viewDU.g.g1 = 101; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: []}); + viewDU.batchHashTreeRoot(); + viewDU.h = listBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then push 1 item to ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: [1]}); + viewDU.batchHashTreeRoot(); + viewDU.h.push(2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: [1, 3]}); + viewDU.batchHashTreeRoot(); + viewDU.h.set(1, 2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h.set(1, 2); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify VectorBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, i: []}); + viewDU.batchHashTreeRoot(); + viewDU.i = vectorBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.i = vectorBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of VectorBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, i: [1, 3]}); + viewDU.batchHashTreeRoot(); + viewDU.i.set(1, 2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.i.set(1, 2); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: []}); + viewDU.batchHashTreeRoot(); + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then push 1 item to ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: []}); + viewDU.batchHashTreeRoot(); + viewDU.j.push(childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.j.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 field of 1 item of ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.j.get(0).f1 = 2; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j.get(0).f1 = 2; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 9, f1: 9}]}); + viewDU.batchHashTreeRoot(); + viewDU.k = vectorCompositeType.toViewDU([{f0: 1, f1: 2}]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k = vectorCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.k.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 field 1 item of VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.k.get(0).f1 = 2; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k.get(0).f1 = 2; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitVectorType", () => { + const viewDU = parentContainerType.toViewDU({...value, l: BitArray.fromSingleBit(64, 4)}); + viewDU.batchHashTreeRoot(); + viewDU.l = bitVectorType.toViewDU(BitArray.fromSingleBit(64, 5)); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.l = bitVectorType.toViewDU(BitArray.fromSingleBit(64, 5)); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitVectorType bit", () => { + const viewDU = parentContainerType.toViewDU({...value, l: BitArray.fromSingleBit(64, 4)}); + viewDU.batchHashTreeRoot(); + viewDU.l.set(4, false); + viewDU.l.set(5, true); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.l.set(4, false); + viewDU.l.set(5, true); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitListType", () => { + const viewDU = parentContainerType.toViewDU({...value, m: BitArray.fromSingleBit(4, 0)}); + viewDU.batchHashTreeRoot(); + viewDU.m = bitListType.toViewDU(BitArray.fromSingleBit(4, 1)); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.m = bitListType.toViewDU(BitArray.fromSingleBit(4, 1)); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitListType bit", () => { + const viewDU = parentContainerType.toViewDU({...value, m: BitArray.fromSingleBit(4, 0)}); + viewDU.batchHashTreeRoot(); + viewDU.m.set(0, false); + viewDU.m.set(1, true); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.m.set(0, false); + viewDU.m.set(1, true); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify OptionalType", () => { + const viewDU = parentContainerType.toViewDU({...value, n: null}); + viewDU.batchHashTreeRoot(); + viewDU.n = listBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.n = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); +}); + +describe("Optional types", () => { + it("offsets are relative to the start of serialized active fields, after the Bitvector[N]", () => { + const parentContainer = new ProfileType( + { + a: new OptionalType(list8Uint64NumInfType), + }, + BitArray.fromBoolArray([true]) + ); + + const value = {a: [1]}; + // 1st byte is for OptionalFields, 2nd byte is the start of fixed parts + // value 4 is relative to 2nd byte + const expectedBytes = new Uint8Array([1, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]); + const data = {uint8Array: expectedBytes, dataView: new DataView(expectedBytes.buffer, 0, expectedBytes.length)}; + const expectedRoot = parentContainer.hashTreeRoot(value); + expect(parentContainer.serialize(value)).to.deep.equal(expectedBytes); + + let viewDU = parentContainer.toViewDU(value); + expect(viewDU.serialize()).to.deep.equal(expectedBytes); + expect(viewDU.batchHashTreeRoot()).to.deep.equal(expectedRoot); + + viewDU = parentContainer.getViewDU(parentContainer.tree_deserializeFromBytes(data, 0, data.uint8Array.length)); + expect(viewDU.serialize()).to.deep.equal(expectedBytes); + expect(viewDU.batchHashTreeRoot()).to.deep.equal(expectedRoot); + + let view = parentContainer.toView(value); + expect(view.serialize()).to.deep.equal(expectedBytes); + expect(view.hashTreeRoot()).to.deep.equal(expectedRoot); + + view = parentContainer.getView( + new Tree(parentContainer.tree_deserializeFromBytes(data, 0, data.uint8Array.length)) + ); + expect(view.serialize()).to.deep.equal(expectedBytes); + expect(view.hashTreeRoot()).to.deep.equal(expectedRoot); + + const value2 = parentContainer.deserialize(expectedBytes); + expect(value2).to.be.deep.equal(value); + expect(parentContainer.serialize(value2)).to.deep.equal(expectedBytes); + expect(parentContainer.hashTreeRoot(value2)).to.deep.equal(expectedRoot); + }); + + it("set null vs 0", () => { + const parentContainer = new ProfileType( + { + a: new OptionalType(uint64NumInfType), + }, + BitArray.fromBoolArray([true]) + ); + + const value = {a: 0}; + const valueNull = {a: null}; + + // value + const value2 = parentContainer.deserialize(parentContainer.serialize(value)); + expect(value2).to.be.deep.equal(value); + const valueNull2 = parentContainer.deserialize(parentContainer.serialize(valueNull)); + expect(valueNull2).to.be.deep.equal(valueNull); + + // ViewDU + const viewDU = parentContainer.toViewDU(value); + expect(viewDU.a).to.be.equal(0); + const viewDUNull = parentContainer.toViewDU(valueNull); + expect(viewDUNull.a).to.be.null; + + // View + const view = parentContainer.toView(value); + expect(view.a).to.be.equal(0); + const viewNull = parentContainer.toView(valueNull); + expect(viewNull.a).to.be.null; + }); +}); + +describe("Profile type merkleization vs StableContainer", () => { + const optionalType = new OptionalType(uint64NumType); + const listBasicType = new ListBasicType(uint64NumType, 10); + + const stableType = new StableContainerType( + { + a: optionalType, + b: uint64NumType, + c: listBasicType, + d: optionalType, + }, + 8 + ); + + const profileType = new ProfileType( + { + b: uint64NumType, + c: listBasicType, + }, + BitArray.fromBoolArray([false, true, true, false, false, false, false, false]) + ); + + const value = {b: 100, c: [10, 200]}; + + it("batchHashTreeRoot()", () => { + const stableViewDU = stableType.toViewDU({...value, a: null, d: null}); + const profileViewDU = profileType.toViewDU(value); + expect(stableViewDU.batchHashTreeRoot()).to.be.deep.equal(profileViewDU.batchHashTreeRoot()); + }); + + it("hashTreeRoot()", () => { + const stableViewDU = stableType.toViewDU({...value, a: null, d: null}); + const profileViewDU = profileType.toViewDU(value); + expect(stableViewDU.hashTreeRoot()).to.be.deep.equal(profileViewDU.hashTreeRoot()); + }); +}); diff --git a/packages/ssz/test/unit/byType/profile/valid.test.ts b/packages/ssz/test/unit/byType/profile/valid.test.ts new file mode 100644 index 00000000..9a1f67e2 --- /dev/null +++ b/packages/ssz/test/unit/byType/profile/valid.test.ts @@ -0,0 +1,49 @@ +import {BitArray, ProfileType, UintNumberType} from "../../../../src"; +import {runTypeTestValid} from "../runTypeTestValid"; + +// taken from eip spec tests + +const uint16 = new UintNumberType(2); +const byteType = new UintNumberType(1); + +const Square = new ProfileType( + { + side: uint16, + color: byteType, + }, + BitArray.fromBoolArray([true, true, false, false]) +); + +const Circle = new ProfileType( + { + color: byteType, + radius: uint16, + }, + BitArray.fromBoolArray([false, true, true, false]) +); + +runTypeTestValid({ + type: Square, + defaultValue: {side: 0, color: 0}, + values: [ + { + id: "circle1-0", + serialized: "0x420001", + json: {side: 0x42, color: 1}, + root: "0xbfdb6fda9d02805e640c0f5767b8d1bb9ff4211498a5e2d7c0f36e1b88ce57ff", + }, + ], +}); + +runTypeTestValid({ + type: Circle, + defaultValue: {color: 0, radius: 0}, + values: [ + { + id: "square1-0", + serialized: "0x014200", + json: {color: 1, radius: 0x42}, + root: "0xf66d2c38c8d2afbd409e86c529dff728e9a4208215ca20ee44e49c3d11e145d8", + }, + ], +}); diff --git a/packages/ssz/test/unit/byType/runTypeProofTest.ts b/packages/ssz/test/unit/byType/runTypeProofTest.ts index aa7a7441..e0b36e2a 100644 --- a/packages/ssz/test/unit/byType/runTypeProofTest.ts +++ b/packages/ssz/test/unit/byType/runTypeProofTest.ts @@ -1,6 +1,15 @@ import {Node} from "@chainsafe/persistent-merkle-tree"; import {expect} from "chai"; -import {BitArray, ContainerType, fromHexString, JsonPath, OptionalType, Type} from "../../../src"; +import { + BitArray, + ContainerType, + fromHexString, + JsonPath, + OptionalType, + ProfileType, + StableContainerType, + Type, +} from "../../../src"; import {CompositeTypeAny, isCompositeType} from "../../../src/type/composite"; import {ArrayBasicTreeView} from "../../../src/view/arrayBasic"; import {RootHex} from "../../lodestarTypes"; @@ -37,9 +46,12 @@ export function runProofTestOnAllJsonPaths({ const viewLeafFromProof = getJsonPathView(type, viewFromProof, jsonPath); const jsonLeaf = getJsonPathValue(type, json, jsonPath); - const jsonLeafFromProof = typeLeaf.toJson( - isCompositeType(typeLeaf) ? typeLeaf.toValueFromView(viewLeafFromProof) : viewLeafFromProof - ); + const jsonLeafFromProof = + viewLeafFromProof == null + ? viewLeafFromProof + : typeLeaf.toJson( + isCompositeType(typeLeaf) ? typeLeaf.toValueFromView(viewLeafFromProof) : viewLeafFromProof + ); expect(jsonLeafFromProof).to.deep.equal(jsonLeaf, "Wrong value fromProof"); @@ -108,9 +120,10 @@ function getJsonPathView(type: Type, view: unknown, jsonPath: JsonPath) if (typeof jsonProp === "number") { view = (view as ArrayBasicTreeView).get(jsonProp); } else if (typeof jsonProp === "string") { - if (type instanceof ContainerType) { + if (type instanceof ContainerType || type instanceof StableContainerType || type instanceof ProfileType) { // Coerce jsonProp to a fieldName. JSON paths may be in JSON notation or fieldName notation - const fieldName = type["jsonKeyToFieldName"][jsonProp] ?? jsonProp; + const fieldName = + (type as ContainerType>>)["jsonKeyToFieldName"][jsonProp] ?? jsonProp; view = (view as Record)[fieldName as string]; } else { throw Error(`type ${type.typeName} is not a ContainerType - jsonProp '${jsonProp}'`); @@ -139,8 +152,8 @@ function getJsonPathValue(type: Type, json: unknown, jsonPath: JsonPath if (typeof jsonProp === "number") { json = (json as unknown[])[jsonProp]; } else if (typeof jsonProp === "string") { - if (type instanceof ContainerType) { - if (type["jsonKeyToFieldName"][jsonProp] === undefined) { + if (type instanceof ContainerType || type instanceof StableContainerType || type instanceof ProfileType) { + if ((type as ContainerType>>)["jsonKeyToFieldName"][jsonProp] === undefined) { throw Error(`Unknown jsonProp ${jsonProp} for type ${type.typeName}`); } diff --git a/packages/ssz/test/unit/byType/stableContainer/tree.test.ts b/packages/ssz/test/unit/byType/stableContainer/tree.test.ts new file mode 100644 index 00000000..36d40dfe --- /dev/null +++ b/packages/ssz/test/unit/byType/stableContainer/tree.test.ts @@ -0,0 +1,723 @@ +import {expect} from "chai"; +import { + BitArray, + BitListType, + BitVectorType, + BooleanType, + ByteListType, + ByteVectorType, + ContainerNodeStructType, + ContainerType, + ListBasicType, + ListCompositeType, + NoneType, + OptionalType, + StableContainerType, + toHexString, + UnionType, + ValueOf, + VectorBasicType, + VectorCompositeType, +} from "../../../../src"; +import {uint64NumInfType, uint64NumType} from "../../../utils/primitiveTypes"; +import {runViewTestMutation} from "../runViewTestMutation"; +import {getNodesAtDepth, Tree, zeroHash} from "@chainsafe/persistent-merkle-tree"; + +// Test both ContainerType, ContainerNodeStructType only if +// - All fields are immutable + +// TODO: test different number of fields to test the serialization + +runViewTestMutation({ + // Use Number64UintType and NumberUintType to test they work the same + type: new StableContainerType({a: uint64NumInfType, b: uint64NumInfType}, 8), + mutations: [ + { + id: "set basic", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 2}, + fn: (tv) => { + tv.a = 10; + }, + }, + { + id: "set basic x2", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 20}, + fn: (tv) => { + tv.a = 10; + tv.b = 20; + }, + }, + // Test that reading a uin64 value that spans two hashObject h values works + // the same with Number64UintType and NumberUintType + { + id: "swap props", + valueBefore: {a: 0xffffffff + 1, b: 0xffffffff + 2}, + valueAfter: {a: 0xffffffff + 2, b: 0xffffffff + 1}, + fn: (tv) => { + const a = tv.a; + const b = tv.b; + tv.a = b; + tv.b = a; + }, + }, + ], +}); + +const containerUintsType = new StableContainerType({a: uint64NumInfType, b: uint64NumInfType}, 8); + +runViewTestMutation({ + type: containerUintsType, + treeViewToStruct: (tv) => ({a: tv.a, b: tv.b}), + mutations: [ + { + id: "set all properties", + valueBefore: {a: 1, b: 2}, + valueAfter: {a: 10, b: 20}, + fn: (tv) => { + tv.a = 10; + tv.b = 21; + // Change twice on purpose to trigger a branch in set basic + tv.b = 20; + }, + }, + ], +}); + +const byte32 = new ByteVectorType(32); +const containerBytesType = new StableContainerType({a: byte32, b: byte32}, 8); +const rootOf = (i: number): Buffer => Buffer.alloc(32, i); + +runViewTestMutation({ + type: containerBytesType, + treeViewToStruct: (tv) => ({a: tv.a, b: tv.b}), + mutations: [ + { + id: "set all properties", + valueBefore: {a: rootOf(1), b: rootOf(2)}, + valueAfter: {a: rootOf(3), b: rootOf(4)}, + fn: (tv) => { + tv.a = rootOf(3); + tv.b = rootOf(4); + }, + }, + ], +}); + +const stableContainerUint64 = new StableContainerType({a: uint64NumType}, 8); + +describe(`${stableContainerUint64.typeName} drop caches`, () => { + it("Make some changes then get previous value", () => { + const view = stableContainerUint64.defaultViewDU(); + const bytesBefore = toHexString(view.serialize()); + + // Make changes to view and clone them to new view + view.a = 1; + view.clone(); + + const bytesAfter = toHexString(view.serialize()); + expect(bytesAfter).to.equal(bytesBefore, "view retained changes"); + }); +}); + +// Test only ContainerType if +// - Some fields are mutable + +const list8Uint64NumInfType = new ListBasicType(uint64NumInfType, 8); + +runViewTestMutation({ + type: new StableContainerType({a: uint64NumInfType, b: uint64NumInfType, list: list8Uint64NumInfType}, 8), + mutations: [ + { + id: "set composite entire list", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + tv.list = list8Uint64NumInfType.toViewDU([10, 20]); + }, + }, + { + id: "set composite list with push", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + tv.list.push(10); + tv.list.push(20); + }, + }, + // Test that keeping a reference to `list` and pushing twice mutates the original tv value + { + id: "set composite list with push and reference", + valueBefore: {a: 1, b: 2, list: []}, + valueAfter: {a: 1, b: 2, list: [10, 20]}, + fn: (tv) => { + const list = tv.list; + list.push(10); + list.push(20); + }, + }, + ], +}); + +const containerUint64 = new StableContainerType({a: uint64NumType}, 8); +const listOfContainers = new ListCompositeType(containerUint64, 4); + +runViewTestMutation({ + // Ensure mutations of child array are commited + type: new ContainerType({list: listOfContainers}), + treeViewToStruct: (tv) => { + const listArr: ValueOf = []; + for (let i = 0; i < tv.list.length; i++) { + const item = tv.list.get(i); + listArr.push({a: item.a}); + } + return {list: listArr}; + }, + mutations: [ + { + id: "Push two values", + valueBefore: {list: []}, + valueAfter: {list: [{a: 1}, {a: 2}]}, + fn: (tv) => { + tv.list.push(containerUint64.toViewDU({a: 1})); + tv.list.push(containerUint64.toViewDU({a: 2})); + }, + }, + ], +}); + +// to test new the VietDU.serialize() implementation for different types +const mixedContainer = new StableContainerType( + { + // a basic type + a: uint64NumType, + // a list basic type + b: new ListBasicType(uint64NumType, 10), + // a list composite type + c: new ListCompositeType(new ContainerType({a: uint64NumInfType, b: uint64NumInfType}), 10), + // embedded container type + d: new ContainerType({a: uint64NumInfType}), + // a union type, cannot mutate through this test + e: new UnionType([new NoneType(), uint64NumInfType]), + }, + 8 +); + +runViewTestMutation({ + type: mixedContainer, + mutations: [ + { + id: "increase by 1", + valueBefore: {a: 10, b: [0, 1], c: [{a: 100, b: 101}], d: {a: 1000}, e: {selector: 1, value: 2000}}, + // View/ViewDU of Union is a value so we cannot mutate + valueAfter: {a: 11, b: [1, 2], c: [{a: 101, b: 102}], d: {a: 1001}, e: {selector: 1, value: 2000}}, + fn: (tv) => { + tv.a += 1; + const b = tv.b; + for (let i = 0; i < b.length; i++) { + b.set(i, b.get(i) + 1); + } + const c = tv.c; + for (let i = 0; i < c.length; i++) { + const item = c.get(i); + item.a += 1; + item.b += 1; + } + tv.d.a += 1; + // does not affect anyway, leaving here to make it explicit + tv.e = {selector: 1, value: tv.e.value ?? 0 + 1}; + }, + }, + ], +}); + +describe("StableContainerViewDU batchHashTreeRoot", function () { + const childContainerType = new ContainerType({f0: uint64NumInfType, f1: uint64NumInfType}); + const unionType = new UnionType([new NoneType(), uint64NumType]); + const listBasicType = new ListBasicType(uint64NumType, 10); + const vectorBasicType = new VectorBasicType(uint64NumType, 2); + const listCompositeType = new ListCompositeType(childContainerType, 10); + const vectorCompositeType = new VectorCompositeType(childContainerType, 1); + const bitVectorType = new BitVectorType(64); + const bitListType = new BitListType(4); + const childContainerStruct = new ContainerNodeStructType({g0: uint64NumInfType, g1: uint64NumInfType}); + const optionalType = new OptionalType(listBasicType); + const parentContainerType = new StableContainerType( + { + a: uint64NumType, + b: new BooleanType(), + c: unionType, + d: new ByteListType(64), + e: new ByteVectorType(64), + // a child container type + f: childContainerType, + g: childContainerStruct, + h: listBasicType, + i: vectorBasicType, + j: listCompositeType, + k: vectorCompositeType, + l: bitVectorType, + m: bitListType, + n: optionalType, + }, + 64 + ); + + const value: ValueOf = { + a: 10, + b: true, + c: {selector: 1, value: 100}, + d: Buffer.alloc(64, 2), + e: Buffer.alloc(64, 1), + f: {f0: 100, f1: 101}, + g: {g0: 100, g1: 101}, + h: [1, 2], + i: [1, 2], + j: [{f0: 1, f1: 2}], + k: [{f0: 1, f1: 2}], + l: BitArray.fromSingleBit(64, 5), + m: BitArray.fromSingleBit(4, 1), + n: [1, 2], + }; + const expectedRoot = parentContainerType.hashTreeRoot(value); + + it("fresh ViewDU", () => { + expect(parentContainerType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify Number type", () => { + const viewDU = parentContainerType.toViewDU({...value, a: 9}); + viewDU.batchHashTreeRoot(); + viewDU.a += 1; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.a = 10; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BooleanType", () => { + const viewDU = parentContainerType.toViewDU({...value, b: false}); + viewDU.batchHashTreeRoot(); + viewDU.b = true; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.b = true; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify UnionType", () => { + const viewDU = parentContainerType.toViewDU({...value, c: {selector: 1, value: 101}}); + viewDU.batchHashTreeRoot(); + viewDU.c = unionType.toViewDU({selector: 1, value: 100}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.c = unionType.toViewDU({selector: 1, value: 100}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ByteVectorType", () => { + const viewDU = parentContainerType.toViewDU(value); + viewDU.batchHashTreeRoot(); + // this takes more than 1 chunk so the resulting node is a branch node + viewDU.e = viewDU.e.slice(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.e = viewDU.e.slice(); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ByteListType", () => { + const viewDU = parentContainerType.toViewDU(value); + viewDU.batchHashTreeRoot(); + // this takes more than 1 chunk so the resulting node is a branch node + viewDU.d = viewDU.d.slice(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.d = viewDU.d.slice(); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify full child container", () => { + const viewDU = parentContainerType.toViewDU({...value, f: {f0: 99, f1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.f = childContainerType.toViewDU({f0: 100, f1: 101}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.f = childContainerType.toViewDU({f0: 100, f1: 101}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify partial child container", () => { + const viewDU = parentContainerType.toViewDU({...value, f: {f0: 99, f1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.f.f0 = 100; + viewDU.f.f1 = 101; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.f.f0 = 100; + viewDU.f.f1 = 101; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ContainerNodeStructType", () => { + const viewDU = parentContainerType.toViewDU({...value, g: {g0: 99, g1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.g = childContainerStruct.toViewDU({g0: 100, g1: 101}); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.g = childContainerStruct.toViewDU({g0: 100, g1: 101}); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify partial ContainerNodeStructType", () => { + const viewDU = parentContainerType.toViewDU({...value, g: {g0: 99, g1: 999}}); + viewDU.batchHashTreeRoot(); + viewDU.g.g0 = 100; + viewDU.g.g1 = 101; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.g.g0 = 100; + viewDU.g.g1 = 101; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: []}); + viewDU.batchHashTreeRoot(); + viewDU.h = listBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then push 1 item to ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: [1]}); + viewDU.batchHashTreeRoot(); + viewDU.h.push(2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of ListBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, h: [1, 3]}); + viewDU.batchHashTreeRoot(); + viewDU.h.set(1, 2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.h.set(1, 2); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify VectorBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, i: []}); + viewDU.batchHashTreeRoot(); + viewDU.i = vectorBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.i = vectorBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of VectorBasicType", () => { + const viewDU = parentContainerType.toViewDU({...value, i: [1, 3]}); + viewDU.batchHashTreeRoot(); + viewDU.i.set(1, 2); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.i.set(1, 2); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: []}); + viewDU.batchHashTreeRoot(); + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then push 1 item to ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: []}); + viewDU.batchHashTreeRoot(); + viewDU.j.push(childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j = listCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.j.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 field of 1 item of ListCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, j: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.j.get(0).f1 = 2; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.j.get(0).f1 = 2; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 9, f1: 9}]}); + viewDU.batchHashTreeRoot(); + viewDU.k = vectorCompositeType.toViewDU([{f0: 1, f1: 2}]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k = vectorCompositeType.toViewDU([{f0: 1, f1: 2}]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 item of VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.k.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k.set(0, childContainerType.toViewDU({f0: 1, f1: 2})); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify 1 field 1 item of VectorCompositeType", () => { + const viewDU = parentContainerType.toViewDU({...value, k: [{f0: 1, f1: 3}]}); + viewDU.batchHashTreeRoot(); + viewDU.k.get(0).f1 = 2; + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.k.get(0).f1 = 2; + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitVectorType", () => { + const viewDU = parentContainerType.toViewDU({...value, l: BitArray.fromSingleBit(64, 4)}); + viewDU.batchHashTreeRoot(); + viewDU.l = bitVectorType.toViewDU(BitArray.fromSingleBit(64, 5)); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.l = bitVectorType.toViewDU(BitArray.fromSingleBit(64, 5)); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitVectorType bit", () => { + const viewDU = parentContainerType.toViewDU({...value, l: BitArray.fromSingleBit(64, 4)}); + viewDU.batchHashTreeRoot(); + viewDU.l.set(4, false); + viewDU.l.set(5, true); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.l.set(4, false); + viewDU.l.set(5, true); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitListType", () => { + const viewDU = parentContainerType.toViewDU({...value, m: BitArray.fromSingleBit(4, 0)}); + viewDU.batchHashTreeRoot(); + viewDU.m = bitListType.toViewDU(BitArray.fromSingleBit(4, 1)); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.m = bitListType.toViewDU(BitArray.fromSingleBit(4, 1)); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify BitListType bit", () => { + const viewDU = parentContainerType.toViewDU({...value, m: BitArray.fromSingleBit(4, 0)}); + viewDU.batchHashTreeRoot(); + viewDU.m.set(0, false); + viewDU.m.set(1, true); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.m.set(0, false); + viewDU.m.set(1, true); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); + + it("full hash then modify OptionalType", () => { + const viewDU = parentContainerType.toViewDU({...value, n: null}); + viewDU.batchHashTreeRoot(); + viewDU.n = listBasicType.toViewDU([1, 2]); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + // assign again but commit before batchHashTreeRoot() + viewDU.n = listBasicType.toViewDU([1, 2]); + viewDU.commit(); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + }); +}); + +describe("StableContainer BitVector[N]", () => { + const optionalType = new OptionalType(uint64NumType); + + it("should have correct serialized size", () => { + for (const maxFields of [8, 16, 64, 256]) { + const stableType = new StableContainerType({a: optionalType}, maxFields); + // should prepend with BitVector[N] + const bytesLength = Math.ceil(maxFields / 8); + expect(stableType.defaultView().serialize().length).to.be.equal(bytesLength); + expect(stableType.defaultViewDU().serialize().length).to.be.equal(bytesLength); + expect(stableType.serialize(stableType.defaultValue()).length).to.be.equal(bytesLength); + } + }); + + it("should have correct active_fields", () => { + for (const maxFields of [64, 256, 512, 1024]) { + const stableType = new StableContainerType({a: optionalType}, maxFields); + for (const view of [stableType.defaultView(), stableType.defaultViewDU()]) { + view.a = 1; + // commit() inside ViewDU + view.hashTreeRoot(); + const activeFieldsDepth = Math.ceil(Math.log2(Math.ceil(maxFields / 256))); + const activeFieldsRootNodes = getNodesAtDepth(view.node.right, activeFieldsDepth, 0, 4); + let isFirst = true; + for (const node of activeFieldsRootNodes) { + if (isFirst) { + const root = node.root; + expect(root[0]).to.be.equal(1); + root[0] = 0; + expect(root).to.deep.equal(zeroHash(0)); + } else { + expect(node.root).to.deep.equal(zeroHash(0)); + } + isFirst = false; + } + + expect(view.node.root).to.be.deep.equal(stableType.hashTreeRoot({a: 1})); + } + } + }); + + it("offsets are relative to the start of serialized active fields, after the Bitvector[N]", () => { + const stableType = new StableContainerType({a: new OptionalType(list8Uint64NumInfType)}, 64); + const value = {a: [1]}; + // first 8 bytes are the active fields + // value 4 at 9th byte is relative to the 9th byte + const expectedBytes = new Uint8Array([1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]); + const data = {uint8Array: expectedBytes, dataView: new DataView(expectedBytes.buffer, 0, expectedBytes.length)}; + const expectedRoot = stableType.hashTreeRoot(value); + expect(stableType.serialize(value)).to.be.deep.equal(expectedBytes); + + let viewDU = stableType.toViewDU(value); + expect(viewDU.serialize()).to.be.deep.equal(expectedBytes); + expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); + + viewDU = stableType.getViewDU(stableType.tree_deserializeFromBytes(data, 0, data.uint8Array.length)); + expect(viewDU.serialize()).to.deep.equal(expectedBytes); + expect(viewDU.batchHashTreeRoot()).to.deep.equal(expectedRoot); + + let view = stableType.toView(value); + expect(view.serialize()).to.deep.equal(expectedBytes); + expect(view.hashTreeRoot()).to.deep.equal(expectedRoot); + + view = stableType.getView(new Tree(stableType.tree_deserializeFromBytes(data, 0, data.uint8Array.length))); + expect(view.serialize()).to.deep.equal(expectedBytes); + expect(view.hashTreeRoot()).to.deep.equal(expectedRoot); + + const value2 = stableType.deserialize(expectedBytes); + expect(value2).to.be.deep.equal(value); + expect(stableType.serialize(value2)).to.deep.equal(expectedBytes); + expect(stableType.hashTreeRoot(value2)).to.deep.equal(expectedRoot); + }); +}); + +describe("StableContainer backward compatibility", () => { + it("add 1 optional field", () => { + const optionalType = new OptionalType(uint64NumType); + const listBasicType = new ListBasicType(uint64NumType, 10); + const Type1 = new StableContainerType( + { + a: optionalType, + b: optionalType, + c: listBasicType, + }, + 8 + ); + + // grow the container with type c + const Type2 = new StableContainerType( + { + a: optionalType, + b: optionalType, + c: listBasicType, + d: optionalType, + }, + 8 + ); + + const value = {a: null, b: 2, c: [1, 2], d: null}; + const viewDU2 = Type2.toViewDU(value); + const serialized = viewDU2.serialize(); + const byteView = { + uint8Array: serialized, + dataView: new DataView(serialized.buffer, serialized.byteOffset, serialized.byteLength), + }; + // can deserialize + const node = Type1.tree_deserializeFromBytes(byteView, 0, serialized.length); + const viewDU = Type2.getViewDU(node); + // hashTreeRoot is the same + expect(viewDU.hashTreeRoot()).to.deep.equal(viewDU2.hashTreeRoot()); + }); +}); diff --git a/packages/ssz/test/unit/byType/stableContainer/valid.test.ts b/packages/ssz/test/unit/byType/stableContainer/valid.test.ts new file mode 100644 index 00000000..90bfe688 --- /dev/null +++ b/packages/ssz/test/unit/byType/stableContainer/valid.test.ts @@ -0,0 +1,132 @@ +import {ListBasicType, OptionalType, StableContainerType, UintNumberType} from "../../../../src"; +import {runTypeTestValid} from "../runTypeTestValid"; + +// taken from eip spec tests + +const optionalUint16 = new OptionalType(new UintNumberType(2)); +const byteType = new UintNumberType(1); +const Shape1 = new StableContainerType( + { + side: optionalUint16, + color: byteType, + radius: optionalUint16, + }, + 4 +); + +const Shape2 = new StableContainerType( + { + side: optionalUint16, + color: byteType, + radius: optionalUint16, + }, + 8 +); + +const Shape3 = new StableContainerType( + { + side: optionalUint16, + colors: new OptionalType(new ListBasicType(byteType, 4)), + radius: optionalUint16, + }, + 8 +); + +runTypeTestValid({ + type: Shape1, + defaultValue: {side: null, color: 0, radius: null}, + values: [ + { + id: "shape1-0", + serialized: "0x074200014200", + json: {side: 0x42, color: 1, radius: 0x42}, + root: "0x37b28eab19bc3e246e55d2e2b2027479454c27ee006d92d4847c84893a162e6d", + }, + { + id: "shape1-1", + serialized: "0x03420001", + json: {side: 0x42, color: 1, radius: null}, + root: "0xbfdb6fda9d02805e640c0f5767b8d1bb9ff4211498a5e2d7c0f36e1b88ce57ff", + }, + { + id: "shape1-2", + serialized: "0x0201", + json: {side: null, color: 1, radius: null}, + root: "0x522edd7309c0041b8eb6a218d756af558e9cf4c816441ec7e6eef42dfa47bb98", + }, + { + id: "shape1-3", + serialized: "0x06014200", + json: {side: null, color: 1, radius: 0x42}, + root: "0xf66d2c38c8d2afbd409e86c529dff728e9a4208215ca20ee44e49c3d11e145d8", + }, + ], +}); +// +runTypeTestValid({ + type: Shape2, + defaultValue: {side: null, color: 0, radius: null}, + values: [ + { + id: "shape2-0", + serialized: "0x074200014200", + json: {side: 0x42, color: 1, radius: 0x42}, + root: "0x0792fb509377ee2ff3b953dd9a88eee11ac7566a8df41c6c67a85bc0b53efa4e", + }, + { + id: "shape2-1", + serialized: "0x03420001", + json: {side: 0x42, color: 1, radius: null}, + root: "0xddc7acd38ae9d6d6788c14bd7635aeb1d7694768d7e00e1795bb6d328ec14f28", + }, + { + id: "shape2-2", + serialized: "0x0201", + json: {side: null, color: 1, radius: null}, + root: "0x9893ecf9b68030ff23c667a5f2e4a76538a8e2ab48fd060a524888a66fb938c9", + }, + { + id: "shape2-3", + serialized: "0x06014200", + json: {side: null, color: 1, radius: 0x42}, + root: "0xe823471310312d52aa1135d971a3ed72ba041ade3ec5b5077c17a39d73ab17c5", + }, + ], +}); + +runTypeTestValid({ + type: Shape3, + defaultValue: {side: null, colors: null, radius: null}, + values: [ + { + id: "shape2-0", + serialized: "0x0742000800000042000102", + json: {side: 0x42, colors: [1, 2], radius: 0x42}, + root: "0x1093b0f1d88b1b2b458196fa860e0df7a7dc1837fe804b95d664279635cb302f", + }, + { + id: "shape2-1", + serialized: "0x014200", + json: {side: 0x42, colors: null, radius: null}, + root: "0x28df3f1c3eebd92504401b155c5cfe2f01c0604889e46ed3d22a3091dde1371f", + }, + { + id: "shape2-2", + serialized: "0x02040000000102", + json: {side: null, colors: [1, 2], radius: null}, + root: "0x659638368467b2c052ca698fcb65902e9b42ce8e94e1f794dd5296ceac2dec3e", + }, + { + id: "shape2-3", + serialized: "0x044200", + json: {side: null, colors: null, radius: 0x42}, + root: "0xd585dd0561c718bf4c29e4c1bd7d4efd4a5fe3c45942a7f778acb78fd0b2a4d2", + }, + { + id: "shape2-4", + serialized: "0x060600000042000102", + json: {side: null, colors: [1, 2], radius: 0x42}, + root: "0x00fc0cecc200a415a07372d5d5b8bc7ce49f52504ed3da0336f80a26d811c7bf", + }, + ], +});