diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index d3fff27c7cb0..cdd2bb8b1535 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -62,7 +62,6 @@ "@chainsafe/bls": "^8.1.0", "@chainsafe/blst": "^1.0.0", "@chainsafe/persistent-merkle-tree": "^0.7.1", - "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.16.0", "@lodestar/config": "^1.18.1", "@lodestar/params": "^1.18.1", diff --git a/packages/state-transition/src/cache/effectiveBalanceIncrements.ts b/packages/state-transition/src/cache/effectiveBalanceIncrements.ts index a82eb0300432..bd72b333a03c 100644 --- a/packages/state-transition/src/cache/effectiveBalanceIncrements.ts +++ b/packages/state-transition/src/cache/effectiveBalanceIncrements.ts @@ -3,18 +3,17 @@ import {BeaconStateAllForks} from "../types.js"; /** * Alias to allow easier refactoring. - * TODO: Estimate the risk of future proof of MAX_EFFECTIVE_BALANCE_INCREMENT < 255 */ -export type EffectiveBalanceIncrements = Uint8Array; +export type EffectiveBalanceIncrements = Uint16Array; -/** Helper to prevent re-writting tests downstream if we change Uint8Array to number[] */ +/** Helper to prevent re-writting tests downstream if we change Uint16Array to number[] */ export function getEffectiveBalanceIncrementsZeroed(len: number): EffectiveBalanceIncrements { - return new Uint8Array(len); + return new Uint16Array(len); } /** * effectiveBalanceIncrements length will always be equal or greater than validatorCount. The - * getEffectiveBalanceIncrementsByteLen() modulo is used to reduce the frequency at which its Uint8Array is recreated. + * getEffectiveBalanceIncrementsByteLen() modulo is used to reduce the frequency at which its Uint16Array is recreated. * if effectiveBalanceIncrements has length greater than validatorCount it's not a problem since those values would * never be accessed. */ @@ -22,7 +21,7 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef // TODO: Research what's the best number to minimize both memory cost and copy costs const byteLen = 1024 * Math.ceil(validatorCount / 1024); - return new Uint8Array(byteLen); + return new Uint16Array(byteLen); } /** @@ -32,7 +31,7 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef */ export function getEffectiveBalanceIncrements(state: BeaconStateAllForks): EffectiveBalanceIncrements { const validatorsArr = state.validators.getAllReadonlyValues(); - const effectiveBalanceIncrements = new Uint8Array(validatorsArr.length); + const effectiveBalanceIncrements = new Uint16Array(validatorsArr.length); for (let i = 0; i < validatorsArr.length; i++) { effectiveBalanceIncrements[i] = Math.floor(validatorsArr[i].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index af1362b55a7e..b03576a653ec 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -647,8 +647,8 @@ export class EpochCache { beforeEpochTransition(): void { // Clone (copy) before being mutated in processEffectiveBalanceUpdates - // NOTE: Force to use Uint8Array.slice (copy) instead of Buffer.call (not copy) - this.effectiveBalanceIncrements = Uint8Array.prototype.slice.call(this.effectiveBalanceIncrements, 0); + // NOTE: Force to use Uint16Array.slice (copy) instead of Buffer.call (not copy) + this.effectiveBalanceIncrements = Uint16Array.prototype.slice.call(this.effectiveBalanceIncrements, 0); } /** @@ -1042,13 +1042,13 @@ export class EpochCache { const newLength = index >= this.effectiveBalanceIncrements.length ? index + 1 : this.effectiveBalanceIncrements.length; const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(newLength); + this.effectiveBalanceIncrements = new Uint16Array(newLength); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); } else { if (index >= this.effectiveBalanceIncrements.length) { // Clone and extend effectiveBalanceIncrements const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements = new Uint16Array(getEffectiveBalanceIncrementsByteLen(index + 1)); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); } } diff --git a/packages/state-transition/src/util/balance.ts b/packages/state-transition/src/util/balance.ts index e305c745ab72..e9b7a06e4130 100644 --- a/packages/state-transition/src/util/balance.ts +++ b/packages/state-transition/src/util/balance.ts @@ -56,8 +56,8 @@ export function getEffectiveBalanceIncrementsZeroInactive( const validatorCount = justifiedState.validators.length; const {effectiveBalanceIncrements} = justifiedState.epochCtx; // Slice up to `validatorCount` since it won't be mutated, nor accessed beyond `validatorCount` - // NOTE: Force to use Uint8Array.slice (copy) instead of Buffer.call (not copy) - const effectiveBalanceIncrementsZeroInactive = Uint8Array.prototype.slice.call( + // NOTE: Force to use Uint16Array.slice (copy) instead of Buffer.call (not copy) + const effectiveBalanceIncrementsZeroInactive = Uint16Array.prototype.slice.call( effectiveBalanceIncrements, 0, validatorCount diff --git a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts b/packages/state-transition/test/memory/effectiveBalanceIncrements.ts deleted file mode 100644 index f1c603b85657..000000000000 --- a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts +++ /dev/null @@ -1,62 +0,0 @@ -import {MutableVector} from "@chainsafe/persistent-ts"; -import {testRunnerMemory} from "@lodestar/beacon-node/test/memory/testRunnerMemory"; -import {newZeroedArray} from "../../src/index.js"; - -// Results in Linux Feb 2022 -// -// EffectiveBalanceIncrements Uint8Array 300000 - 299873.5 bytes / instance -// EffectiveBalanceIncrements array 300000 - 2400093.1 bytes / instance -// EffectiveBalanceIncrements MutableVector 300000 - 4380557.0 bytes / instance -// EffectiveBalanceIncrements MutableVector 300000 cloned 10 - 4399575.0 bytes / instance -// -// With MutableVector, break even at 14 instances of Uint8Array -// 4380557 / 299873 = 14 - -const vc = 300_000; -const cloneTimes = 10; - -testRunnerMemoryBpi([ - { - id: `EffectiveBalanceIncrements Uint8Array ${vc}`, - getInstance: () => new Uint8Array(vc), - }, - { - id: `EffectiveBalanceIncrements array ${vc}`, - getInstance: () => newZeroedArray(vc), - }, - { - id: `EffectiveBalanceIncrements MutableVector ${vc}`, - getInstance: () => MutableVector.from(newZeroedArray(vc)), - }, - { - id: `EffectiveBalanceIncrements MutableVector ${vc} cloned ${cloneTimes}`, - getInstance: () => { - const mv = MutableVector.from(newZeroedArray(vc)); - const mvs = [mv]; - for (let i = 0; i < cloneTimes; i++) { - const mvc = mv.clone(); - mvc.push(0); - mvs.push(mvc); - } - return mvs; - }, - }, -]); - -/** - * Test bytes per instance in different representations of raw binary data - */ -function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { - const longestId = Math.max(...testCases.map(({id}) => id.length)); - - for (const {id, getInstance} of testCases) { - const bpi = testRunnerMemory({ - getInstance, - convergeFactor: 1 / 100, - sampleEvery: 5, - }); - - // eslint-disable-next-line no-console - console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); - } -} diff --git a/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts b/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts index 1f4141912627..7e10f447181f 100644 --- a/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts +++ b/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts @@ -1,5 +1,3 @@ -import {MutableVector} from "@chainsafe/persistent-ts"; - const refs: any[] = []; const xs: number[] = []; const arrayBuffersArr: number[] = []; @@ -23,7 +21,6 @@ const size = 100; const testType = TestType.Set; let arrayNumGlobal: number[] | null = null; -let mutableVectorGlobal: MutableVector | null = null; for (let i = 0; i < 1e8; i++) { switch (testType as TestType) { @@ -65,49 +62,6 @@ for (let i = 0; i < 1e8; i++) { break; } - // size | 100 | 1000 | 10000 | - // ---- | ------ | ------ | ------ | - // rssM | 1817.4 | 15518. | 154335 | - case TestType.MutableVector: { - const items = createArray(size); - const mutableVector = MutableVector.from(items); - refs.push(mutableVector); - break; - } - - // size | 100 | 1000 | - // ---- | ------ | ------ | - // rssM | 58.68 | 55.89 | - case TestType.MutableVectorClone: { - if (!mutableVectorGlobal) { - const items = createArray(size); - mutableVectorGlobal = MutableVector.from(items); - } - refs.push(mutableVectorGlobal.clone()); - break; - } - - // Grid of size / changes, all values = rssM in bytes - // | 100 | 1000 | 10000 | - // ----- | ------ | ------ | ------ | - // 1 | 793.45 | 801.53 | 1137.9 | - // 10 | 803.98 | 802.36 | 1144.9 | - // 100 | 1573.2 | 1826.4 | 2172.0 | - // 1000 | - | 11250. | 11886. | - // 10000 | - | - | 111365 | - case TestType.MutableVectorCloneAndMutate: { - if (!mutableVectorGlobal) { - const items = createArray(size); - mutableVectorGlobal = MutableVector.from(items); - } - const newArr = mutableVectorGlobal.clone(); - for (let j = 0; j < 10000; j++) { - newArr.set(j, i); - } - refs.push(newArr); - break; - } - // size | 100 | 1000 | // ---- | ------ | ------ | // rssM | 2646.8 | 20855. | @@ -161,14 +115,6 @@ for (let i = 0; i < 1e8; i++) { } } -function createArray(n: number): number[] { - const items: number[] = []; - for (let i = 0; i < n; i++) { - items.push(i); - } - return items; -} - /** * From https://github.com/simple-statistics/simple-statistics/blob/d0d177baf74976a2421638bce98ab028c5afb537/src/linear_regression.js * diff --git a/packages/state-transition/test/perf/dataStructures/arrayish.test.ts b/packages/state-transition/test/perf/dataStructures/arrayish.test.ts index 59162b6eecca..5b6af0d989b6 100644 --- a/packages/state-transition/test/perf/dataStructures/arrayish.test.ts +++ b/packages/state-transition/test/perf/dataStructures/arrayish.test.ts @@ -1,6 +1,5 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {LeafNode, toGindex, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; // Understand the cost of each array-ish data structure to: // - Get one element @@ -99,48 +98,6 @@ describe("Tree (persistent-merkle-tree)", () => { } }); -describe("MutableVector", () => { - // Don't track regressions in CI - setBenchOpts({noThreshold: true}); - - let items: number[]; - let mutableVector: MutableVector; - - before(function () { - items = createArray(n); - mutableVector = MutableVector.from(items); - }); - - itBench(`MutableVector ${n} create`, () => { - MutableVector.from(items); - }); - - itBench({id: `MutableVector ${n} get(${ih})`, runsFactor}, () => { - for (let i = 0; i < runsFactor; i++) mutableVector.get(ih - i); - }); - - itBench({id: `MutableVector ${n} set(${ih})`, runsFactor}, () => { - for (let i = 0; i < runsFactor; i++) mutableVector.set(ih - i, 10000000); - }); - - itBench(`MutableVector ${n} toArray()`, () => { - mutableVector.toArray(); - }); - - itBench(`MutableVector ${n} iterate all - toArray() + loop`, () => { - const mvArr = mutableVector.toArray(); - for (let i = 0; i < n; i++) { - mvArr[i]; - } - }); - - itBench(`MutableVector ${n} iterate all - get(i)`, () => { - for (let i = 0; i < n; i++) { - mutableVector.get(i); - } - }); -}); - describe("Array", () => { // Don't track regressions in CI setBenchOpts({noThreshold: true}); diff --git a/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts b/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts deleted file mode 100644 index 13c2d982e86b..000000000000 --- a/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {MutableVector} from "@chainsafe/persistent-ts"; -import {newZeroedArray} from "../../../src/index.js"; - -describe("effectiveBalanceIncrements", () => { - setBenchOpts({noThreshold: true}); - - const vc = 300_000; - const uint8Array = new Uint8Array(vc); - const mv = MutableVector.from(newZeroedArray(vc)); - - itBench(`effectiveBalanceIncrements clone Uint8Array ${vc}`, () => { - uint8Array.slice(0); - }); - - itBench(`effectiveBalanceIncrements clone MutableVector ${vc}`, () => { - mv.clone(); - }); - - itBench(`effectiveBalanceIncrements rw all Uint8Array ${vc}`, () => { - for (let i = 0; i < vc; i++) { - uint8Array[i]++; - } - }); - - itBench(`effectiveBalanceIncrements rw all MutableVector ${vc}`, () => { - for (let i = 0; i < vc; i++) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - mv.set(i, mv.get(i)! + 1); - } - }); -}); diff --git a/yarn.lock b/yarn.lock index 1f3ad4bc9138..2490fa03d49f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -472,11 +472,6 @@ "@chainsafe/as-sha256" "^0.4.2" "@noble/hashes" "^1.3.0" -"@chainsafe/persistent-ts@^0.19.1": - version "0.19.1" - resolved "https://registry.npmjs.org/@chainsafe/persistent-ts/-/persistent-ts-0.19.1.tgz" - integrity sha512-fUFFFFxdcpYkMAHnjm83EYL/R/smtVmEkJr3FGSI6dwPk4ue9rXjEHf7FTd3V8AbVOcTJGriN4cYf2V+HOYkjQ== - "@chainsafe/prometheus-gc-stats@^1.0.0": version "1.0.2" resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.2.tgz#585f8f1555251db156d7e50ef8c86dd4f3e78f70" @@ -11867,16 +11862,7 @@ string-argv@~0.3.1: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -13511,16 +13497,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==