From a847f862cbb65a9efa8a4432fddcf90f77d5669d Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:25:28 -0400 Subject: [PATCH 1/9] scaffolding --- package-lock.json | 1 + packages/statemanager/package.json | 1 + .../src/statefulVerkleStateManager.ts | 185 ++++++++++++++++++ packages/statemanager/src/types.ts | 7 +- 4 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 packages/statemanager/src/statefulVerkleStateManager.ts diff --git a/package-lock.json b/package-lock.json index 5f176d23981..917919d9e47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12895,6 +12895,7 @@ "@ethereumjs/rlp": "^5.0.2", "@ethereumjs/trie": "^6.2.1", "@ethereumjs/util": "^9.1.0", + "@ethereumjs/verkle": "^0.1.0", "@js-sdsl/ordered-map": "^4.4.2", "debug": "^4.3.3", "ethereum-cryptography": "^2.2.1", diff --git a/packages/statemanager/package.json b/packages/statemanager/package.json index 11f5aa9b535..d4c191a581d 100644 --- a/packages/statemanager/package.json +++ b/packages/statemanager/package.json @@ -54,6 +54,7 @@ "@ethereumjs/rlp": "^5.0.2", "@ethereumjs/trie": "^6.2.1", "@ethereumjs/util": "^9.1.0", + "@ethereumjs/verkle": "^0.1.0", "@js-sdsl/ordered-map": "^4.4.2", "debug": "^4.3.3", "ethereum-cryptography": "^2.2.1", diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts new file mode 100644 index 00000000000..f0322a7208a --- /dev/null +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -0,0 +1,185 @@ +import { Common, Mainnet } from '@ethereumjs/common' +import { type Account, type Address, MapDB, createAccountFromRLP } from '@ethereumjs/util' +import { VerkleTree } from '@ethereumjs/verkle' +import debugDefault from 'debug' + +import { OriginalStorageCache } from './cache/originalStorageCache.js' +import { modifyAccountFields } from './util.js' + +import type { Caches } from './cache/caches.js' +import type { StatefulVerkleStateManagerOpts } from './types.js' +import type { + AccountFields, + Proof, + StateManagerInterface, + StorageDump, + StorageRange, +} from '@ethereumjs/common' +import type { VerkleCrypto } from '@ethereumjs/util' +import type { Debugger } from 'debug' +export class StatefulVerkleStateManager implements StateManagerInterface { + protected _debug: Debugger + protected _caches?: Caches + + originalStorageCache: OriginalStorageCache + + protected _trie: VerkleTree + + public readonly common: Common + + protected _checkpointCount: number + + protected verkleCrypto: VerkleCrypto + /** + * StateManager is run in DEBUG mode (default: false) + * Taken from DEBUG environment variable + * + * Safeguards on debug() calls are added for + * performance reasons to avoid string literal evaluation + * @hidden + */ + protected readonly DEBUG: boolean = false + + constructor(opts: StatefulVerkleStateManagerOpts) { + // Skip DEBUG calls unless 'ethjs' included in environmental DEBUG variables + // Additional window check is to prevent vite browser bundling (and potentially other) to break + this.DEBUG = + typeof window === 'undefined' ? (process?.env?.DEBUG?.includes('ethjs') ?? false) : false + + this._checkpointCount = 0 + + if (opts.common?.isActivatedEIP(6800) === false) + throw new Error('EIP-6800 required for verkle state management') + + this.common = opts.common ?? new Common({ chain: Mainnet, eips: [6800] }) + this._trie = + opts.trie ?? + new VerkleTree({ verkleCrypto: opts.verkleCrypto, db: new MapDB() }) + this._debug = debugDefault('statemanager:statefulVerkleStatemanager') + this.originalStorageCache = new OriginalStorageCache(this.getStorage.bind(this)) + this._caches = opts.caches + + this.verkleCrypto = opts.verkleCrypto + } + getAccount = async (address: Address): Promise => { + const elem = this._caches?.account?.get(address) + if (elem !== undefined) { + return elem.accountRLP !== undefined ? createAccountFromRLP(elem.accountRLP) : undefined + } + + const rlp = await this._trie.get(address.bytes) + const account = rlp !== undefined ? createAccountFromRLP(rlp) : undefined + if (this.DEBUG) { + this._debug(`Get account ${address} from DB (${account ? 'exists' : 'non-existent'})`) + } + this._caches?.account?.put(address, account) + return account + } + + putAccount = async (address: Address, account?: Account): Promise => { + if (this.DEBUG) { + this._debug( + `Save account address=${address} nonce=${account?.nonce} balance=${ + account?.balance + } contract=${account && account.isContract() ? 'yes' : 'no'} empty=${ + account && account.isEmpty() ? 'yes' : 'no' + }`, + ) + } + if (this._caches?.account === undefined) { + const trie = this._trie + if (account !== undefined) { + await trie.put(address.bytes, account.serialize()) + } else { + await trie.del(address.bytes) + } + } else { + if (account !== undefined) { + this._caches.account?.put(address, account) + } else { + this._caches.account?.del(address) + } + } + } + deleteAccount = async (address: Address): Promise => { + if (this.DEBUG) { + this._debug(`Delete account ${address}`) + } + + this._caches?.deleteAccount(address) + + if (this._caches?.account === undefined) { + await this._trie.del(address.bytes) + } + } + + modifyAccountFields = async (address: Address, accountFields: AccountFields): Promise => { + await modifyAccountFields(this, address, accountFields) + } + putCode(address: Address, value: Uint8Array): Promise { + throw new Error('Method not implemented.') + } + getCode(address: Address): Promise { + throw new Error('Method not implemented.') + } + getCodeSize(address: Address): Promise { + throw new Error('Method not implemented.') + } + getStorage(address: Address, key: Uint8Array): Promise { + throw new Error('Method not implemented.') + } + putStorage(address: Address, key: Uint8Array, value: Uint8Array): Promise { + throw new Error('Method not implemented.') + } + clearStorage(address: Address): Promise { + throw new Error('Method not implemented.') + } + checkpoint = async (): Promise => { + this._trie.checkpoint() + this._caches?.checkpoint() + this._checkpointCount++ + } + commit(): Promise { + throw new Error('Method not implemented.') + } + revert(): Promise { + throw new Error('Method not implemented.') + } + getStateRoot(): Promise { + throw new Error('Method not implemented.') + } + setStateRoot(stateRoot: Uint8Array, clearCache?: boolean): Promise { + throw new Error('Method not implemented.') + } + hasStateRoot(root: Uint8Array): Promise { + throw new Error('Method not implemented.') + } + getProof?(address: Address, storageSlots: Uint8Array[]): Promise { + throw new Error('Method not implemented.') + } + dumpStorage?(address: Address): Promise { + throw new Error('Method not implemented.') + } + dumpStorageRange?(address: Address, startKey: bigint, limit: number): Promise { + throw new Error('Method not implemented.') + } + + verifyVerkleProof?(stateRoot: Uint8Array): boolean { + throw new Error('Method not implemented.') + } + verifyPostState?(): boolean { + throw new Error('Method not implemented.') + } + checkChunkWitnessPresent?(contract: Address, programCounter: number): Promise { + throw new Error('Method not implemented.') + } + getAppliedKey?(address: Uint8Array): Uint8Array { + throw new Error('Method not implemented.') + } + clearCaches(): void { + throw new Error('Method not implemented.') + } + shallowCopy(downlevelCaches?: boolean): StateManagerInterface { + throw new Error('Method not implemented.') + } +} diff --git a/packages/statemanager/src/types.ts b/packages/statemanager/src/types.ts index a9051bc2e0e..1b06e8b0c7b 100644 --- a/packages/statemanager/src/types.ts +++ b/packages/statemanager/src/types.ts @@ -4,7 +4,7 @@ import type { AccessWitness, Caches } from './index.js' import type { Common } from '@ethereumjs/common' import type { Trie } from '@ethereumjs/trie' import type { VerkleCrypto } from '@ethereumjs/util' - +import type { VerkleTree } from '@ethereumjs/verkle' /** * Basic state manager options (not to be used directly) */ @@ -75,6 +75,11 @@ export interface StatelessVerkleStateManagerOpts extends BaseStateManagerOpts { caches?: Caches } +export interface StatefulVerkleStateManagerOpts extends BaseStateManagerOpts { + verkleCrypto: VerkleCrypto + trie?: VerkleTree + caches?: Caches +} export interface VerkleState { [key: PrefixedHexString]: PrefixedHexString | null } From 107c88311070f538c16dcea266024ce0f832e010 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:01:30 -0400 Subject: [PATCH 2/9] broken WIP --- .../src/statefulVerkleStateManager.ts | 90 +++++++++++++++---- .../test/statefulVerkleStateManager.spec.ts | 21 +++++ 2 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 packages/statemanager/test/statefulVerkleStateManager.spec.ts diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index f0322a7208a..c87a467da9f 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -1,7 +1,22 @@ import { Common, Mainnet } from '@ethereumjs/common' -import { type Account, type Address, MapDB, createAccountFromRLP } from '@ethereumjs/util' +import { + type Account, + type Address, + MapDB, + VerkleLeafType, + bigIntToBytes, + bytesToBigInt, + bytesToInt32, + createAccountFromRLP, + createPartialAccount, + getVerkleKey, + getVerkleStem, + setLengthRight, + short, +} from '@ethereumjs/util' import { VerkleTree } from '@ethereumjs/verkle' import debugDefault from 'debug' +import { keccak256 } from 'ethereum-cryptography/keccak.js' import { OriginalStorageCache } from './cache/originalStorageCache.js' import { modifyAccountFields } from './util.js' @@ -40,6 +55,8 @@ export class StatefulVerkleStateManager implements StateManagerInterface { */ protected readonly DEBUG: boolean = false + private keccakFunction: Function + constructor(opts: StatefulVerkleStateManagerOpts) { // Skip DEBUG calls unless 'ethjs' included in environmental DEBUG variables // Additional window check is to prevent vite browser bundling (and potentially other) to break @@ -58,21 +75,56 @@ export class StatefulVerkleStateManager implements StateManagerInterface { this._debug = debugDefault('statemanager:statefulVerkleStatemanager') this.originalStorageCache = new OriginalStorageCache(this.getStorage.bind(this)) this._caches = opts.caches - + this.keccakFunction = opts.common?.customCrypto.keccak256 ?? keccak256 this.verkleCrypto = opts.verkleCrypto } + getAccount = async (address: Address): Promise => { const elem = this._caches?.account?.get(address) if (elem !== undefined) { return elem.accountRLP !== undefined ? createAccountFromRLP(elem.accountRLP) : undefined } - const rlp = await this._trie.get(address.bytes) - const account = rlp !== undefined ? createAccountFromRLP(rlp) : undefined + const stem = getVerkleStem(this.verkleCrypto, address, 0) + const versionKey = getVerkleKey(stem, VerkleLeafType.Version) + const balanceKey = getVerkleKey(stem, VerkleLeafType.Balance) + const nonceKey = getVerkleKey(stem, VerkleLeafType.Nonce) + const codeHashKey = getVerkleKey(stem, VerkleLeafType.CodeHash) + const codeSizeKey = getVerkleKey(stem, VerkleLeafType.CodeSize) + const version = await this._trie.get(versionKey) + const balance = await this._trie.get(balanceKey) + const nonce = await this._trie.get(nonceKey) + const codeHash = await this._trie.get(codeHashKey) + // TODO: Only do this check if codeHash !== KECCAK_RLP (or whatever the empty array is) + const codeSize = await this._trie.get(codeSizeKey) + + const account = createPartialAccount({ + version: Array.isArray(version) ? bytesToInt32(version, true) : null, + balance: Array.isArray(balance) ? bytesToBigInt(balance, true) : null, + nonce: Array.isArray(nonce) ? bytesToBigInt(nonce, true) : null, + codeHash: Array.isArray(codeHash) ? codeHash : null, + // if codeSizeRaw is null, it means account didn't exist or it was EOA either way codeSize is 0 + // if codeSizeRaw is undefined, then we pass in null which in our context of partial account means + // not specified + codeSize: Array.isArray(codeSize) ? bytesToInt32(codeSize, true) : null, + storageRoot: null, + }) + // check if the account didn't exist if any of the basic keys are undefined + if ( + version === undefined || + balance === undefined || + nonce === undefined || + codeHash === undefined + ) { + if (this.DEBUG) { + this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) + } + this._caches?.account?.put(address, account) + } + if (this.DEBUG) { - this._debug(`Get account ${address} from DB (${account ? 'exists' : 'non-existent'})`) + this._debug(`getAccount address=${address.toString()} stem=${short(stem)}`) } - this._caches?.account?.put(address, account) return account } @@ -85,19 +137,21 @@ export class StatefulVerkleStateManager implements StateManagerInterface { account && account.isEmpty() ? 'yes' : 'no' }`, ) - } - if (this._caches?.account === undefined) { - const trie = this._trie - if (account !== undefined) { - await trie.put(address.bytes, account.serialize()) - } else { - await trie.del(address.bytes) - } - } else { - if (account !== undefined) { - this._caches.account?.put(address, account) + + if (this._caches?.account === undefined) { + const stem = getVerkleStem(this.verkleCrypto, address, 0) + const balanceKey = getVerkleKey(stem, VerkleLeafType.Balance) + const nonceKey = getVerkleKey(stem, VerkleLeafType.Nonce) + const codeHashKey = getVerkleKey(stem, VerkleLeafType.CodeHash) + + const balanceBuf = setLengthRight(bigIntToBytes(account.balance, true), 32) + const nonceBuf = setLengthRight(bigIntToBytes(account.nonce, true), 32) } else { - this._caches.account?.del(address) + if (account !== undefined) { + this._caches?.account?.put(address, account, true) + } else { + this._caches?.account?.del(address) + } } } } diff --git a/packages/statemanager/test/statefulVerkleStateManager.spec.ts b/packages/statemanager/test/statefulVerkleStateManager.spec.ts new file mode 100644 index 00000000000..4190d1261a0 --- /dev/null +++ b/packages/statemanager/test/statefulVerkleStateManager.spec.ts @@ -0,0 +1,21 @@ +import { VerkleTree } from '@ethereumjs/verkle' +import { describe, it } from 'vitest' + +import { StatefulVerkleStateManager } from '../src/statefulVerkleStateManager.js' + +import type { PrefixedHexString } from '@ethereumjs/util' + +describe('Verkle Tree API tests', () => { + it('should get an account from the trie', async () => { + const trie = await VerkleTree.create() + const sm = new StatefulVerkleStateManager({ trie, verkleCrypto: trie['verkleCrypto'] }) + const accountKeys = [ + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d00', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d03', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d04', + '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d05', + ] as PrefixedHexString[] + }) +}) From 78f199042e957578ba1c9e7c859f9bc128733d8d Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:06:03 -0400 Subject: [PATCH 3/9] partial implementations --- .../src/statefulVerkleStateManager.ts | 110 ++++++++++-------- .../test/statefulVerkleStateManager.spec.ts | 22 ++-- packages/verkle/src/verkleTree.ts | 58 ++++----- packages/verkle/test/verkle.spec.ts | 33 ++++-- 4 files changed, 130 insertions(+), 93 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index c87a467da9f..67bee7f2c98 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -14,7 +14,12 @@ import { setLengthRight, short, } from '@ethereumjs/util' -import { VerkleTree } from '@ethereumjs/verkle' +import { + LeafNode, + VerkleLeafNodeValue, + VerkleTree, + createUntouchedLeafValue, +} from '@ethereumjs/verkle' import debugDefault from 'debug' import { keccak256 } from 'ethereum-cryptography/keccak.js' @@ -86,46 +91,42 @@ export class StatefulVerkleStateManager implements StateManagerInterface { } const stem = getVerkleStem(this.verkleCrypto, address, 0) - const versionKey = getVerkleKey(stem, VerkleLeafType.Version) - const balanceKey = getVerkleKey(stem, VerkleLeafType.Balance) - const nonceKey = getVerkleKey(stem, VerkleLeafType.Nonce) - const codeHashKey = getVerkleKey(stem, VerkleLeafType.CodeHash) - const codeSizeKey = getVerkleKey(stem, VerkleLeafType.CodeSize) - const version = await this._trie.get(versionKey) - const balance = await this._trie.get(balanceKey) - const nonce = await this._trie.get(nonceKey) - const codeHash = await this._trie.get(codeHashKey) - // TODO: Only do this check if codeHash !== KECCAK_RLP (or whatever the empty array is) - const codeSize = await this._trie.get(codeSizeKey) - - const account = createPartialAccount({ - version: Array.isArray(version) ? bytesToInt32(version, true) : null, - balance: Array.isArray(balance) ? bytesToBigInt(balance, true) : null, - nonce: Array.isArray(nonce) ? bytesToBigInt(nonce, true) : null, - codeHash: Array.isArray(codeHash) ? codeHash : null, - // if codeSizeRaw is null, it means account didn't exist or it was EOA either way codeSize is 0 - // if codeSizeRaw is undefined, then we pass in null which in our context of partial account means - // not specified - codeSize: Array.isArray(codeSize) ? bytesToInt32(codeSize, true) : null, - storageRoot: null, - }) - // check if the account didn't exist if any of the basic keys are undefined - if ( - version === undefined || - balance === undefined || - nonce === undefined || - codeHash === undefined - ) { - if (this.DEBUG) { - this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) + + // First retrieve the leaf node from the trie + const leaf = await this._trie.findPath(stem) + if (leaf.node !== null && leaf.node instanceof LeafNode) { + // We have a leaf node so let's pull out the important values for an account + const version = leaf.node.getValue(VerkleLeafType.Version) + const balance = leaf.node.getValue(VerkleLeafType.Balance) + const nonce = leaf.node.getValue(VerkleLeafType.Nonce) + const codeHash = leaf.node.getValue(VerkleLeafType.CodeHash) + const codeSize = leaf.node.getValue(VerkleLeafType.CodeSize) + const account = createPartialAccount({ + version: Array.isArray(version) ? bytesToInt32(version, true) : null, + balance: Array.isArray(balance) ? bytesToBigInt(balance, true) : null, + nonce: Array.isArray(nonce) ? bytesToBigInt(nonce, true) : null, + codeHash: Array.isArray(codeHash) ? codeHash : null, + codeSize: Array.isArray(codeSize) ? bytesToInt32(codeSize, true) : null, + storageRoot: null, + }) + // check if the account didn't exist if any of the basic keys are undefined + if ( + version === undefined || + balance === undefined || + nonce === undefined || + codeHash === undefined + ) { + if (this.DEBUG) { + this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) + } + this._caches?.account?.put(address, account) } - this._caches?.account?.put(address, account) - } - if (this.DEBUG) { - this._debug(`getAccount address=${address.toString()} stem=${short(stem)}`) + if (this.DEBUG) { + this._debug(`getAccount address=${address.toString()} stem=${short(stem)}`) + } + return account } - return account } putAccount = async (address: Address, account?: Account): Promise => { @@ -139,13 +140,28 @@ export class StatefulVerkleStateManager implements StateManagerInterface { ) if (this._caches?.account === undefined) { - const stem = getVerkleStem(this.verkleCrypto, address, 0) - const balanceKey = getVerkleKey(stem, VerkleLeafType.Balance) - const nonceKey = getVerkleKey(stem, VerkleLeafType.Nonce) - const codeHashKey = getVerkleKey(stem, VerkleLeafType.CodeHash) - - const balanceBuf = setLengthRight(bigIntToBytes(account.balance, true), 32) - const nonceBuf = setLengthRight(bigIntToBytes(account.nonce, true), 32) + if (account !== undefined) { + const stem = getVerkleStem(this.verkleCrypto, address, 0) + await this._trie.put( + stem, + [ + VerkleLeafType.Version, + VerkleLeafType.Balance, + VerkleLeafType.Nonce, + VerkleLeafType.CodeHash, + ], + [ + account.balance !== null + ? setLengthRight(bigIntToBytes(account.balance, true), 32) + : createUntouchedLeafValue(), + account.balance !== null + ? setLengthRight(bigIntToBytes(account.balance, true), 32) + : createUntouchedLeafValue(), + setLengthRight(bigIntToBytes(account.nonce, true), 32), + account.codeHash, + ], + ) + } } else { if (account !== undefined) { this._caches?.account?.put(address, account, true) @@ -155,6 +171,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface { } } } + deleteAccount = async (address: Address): Promise => { if (this.DEBUG) { this._debug(`Delete account ${address}`) @@ -163,7 +180,8 @@ export class StatefulVerkleStateManager implements StateManagerInterface { this._caches?.deleteAccount(address) if (this._caches?.account === undefined) { - await this._trie.del(address.bytes) + // TODO: Delete account + // await this._trie.del(address.bytes) } } diff --git a/packages/statemanager/test/statefulVerkleStateManager.spec.ts b/packages/statemanager/test/statefulVerkleStateManager.spec.ts index 4190d1261a0..733a7f04d13 100644 --- a/packages/statemanager/test/statefulVerkleStateManager.spec.ts +++ b/packages/statemanager/test/statefulVerkleStateManager.spec.ts @@ -1,21 +1,17 @@ -import { VerkleTree } from '@ethereumjs/verkle' -import { describe, it } from 'vitest' +import { createAccount, createAddressFromPrivateKey, randomBytes } from '@ethereumjs/util' +import { createVerkleTree } from '@ethereumjs/verkle' +import { assert, describe, it } from 'vitest' import { StatefulVerkleStateManager } from '../src/statefulVerkleStateManager.js' -import type { PrefixedHexString } from '@ethereumjs/util' - describe('Verkle Tree API tests', () => { it('should get an account from the trie', async () => { - const trie = await VerkleTree.create() + const trie = await createVerkleTree() const sm = new StatefulVerkleStateManager({ trie, verkleCrypto: trie['verkleCrypto'] }) - const accountKeys = [ - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d00', - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01', - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d03', - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d04', - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d05', - ] as PrefixedHexString[] + const address = createAddressFromPrivateKey(randomBytes(32)) + const account = createAccount({ nonce: 3n, balance: 0xfffn }) + await sm.putAccount(address, account) + const retrievedAccount = await sm.getAccount(address) + assert.equal(retrievedAccount?.balance, account.balance) }) }) diff --git a/packages/verkle/src/verkleTree.ts b/packages/verkle/src/verkleTree.ts index 7682a912f1a..8c6ca6f015d 100644 --- a/packages/verkle/src/verkleTree.ts +++ b/packages/verkle/src/verkleTree.ts @@ -175,16 +175,19 @@ export class VerkleTree { } /** - * Stores a given `value` at the given `key` or do a delete if `value` is empty Uint8Array - * @param key - the key to store the value at - * @param value - the value to store - * @returns A Promise that resolves once value is stored. + * Stores given `values` at the given `stem` and `suffixes` or do a delete if `value` is empty Uint8Array + * @param key - the stem to store the value at (must be 31 bytes long) + * @param suffixes - array of suffixes at which to store individual values + * @param value - the value(s) to store + * @returns A Promise that resolves once value(s) are stored. */ - async put(key: Uint8Array, value: Uint8Array): Promise { - if (key.length !== 32) throw new Error(`expected key with length 32; got ${key.length}`) - const stem = key.slice(0, 31) - const suffix = key[key.length - 1] - this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}; Suffix: ${suffix}`, ['PUT']) + async put(stem: Uint8Array, suffixes: number[], values: Uint8Array[]): Promise { + if (stem.length !== 31) throw new Error(`expected stem with length 31, got ${stem.length}`) + if (values.length !== suffixes.length) { + // Must have an equal number of values and suffixes + throw new Error(`expected number of values; ${values.length} to equal ${suffixes.length}`) + } + this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}`, ['PUT']) const putStack: [Uint8Array, VerkleNode][] = [] // Find path to nearest node @@ -219,19 +222,24 @@ export class VerkleTree { leafNode = await LeafNode.create(stem, this.verkleCrypto) this.DEBUG && this.debug(`Creating new leaf node at stem: ${bytesToHex(stem)}`, ['PUT']) } - // Update value in leaf node and push to putStack - if (equalsBytes(value, createDeletedLeafValue())) { - // Special case for when the deleted leaf value or zeroes is passed to `put` - // Writing the deleted leaf value to the suffix indicated in the key - leafNode.setValue(suffix, VerkleLeafNodeValue.Deleted) - } else { - leafNode.setValue(suffix, value) + for (let i = 0; i < values.length; i++) { + const value = values[i] + const suffix = suffixes[i] + // Update value(s) in leaf node + if (equalsBytes(value, createDeletedLeafValue())) { + // Special case for when the deleted leaf value or zeroes is passed to `put` + // Writing the deleted leaf value to the suffix + leafNode.setValue(suffix, VerkleLeafNodeValue.Deleted) + } else { + leafNode.setValue(suffix, value) + } + this.DEBUG && + this.debug( + `Updating value for suffix: ${suffix} at leaf node with stem: ${bytesToHex(stem)}`, + ['PUT'], + ) } - this.DEBUG && - this.debug( - `Updating value for suffix: ${suffix} at leaf node with stem: ${bytesToHex(stem)}`, - ['PUT'], - ) + // Push new/updated leafNode to putStack putStack.push([leafNode.hash(), leafNode]) // `path` is the path to the last node pushed to the `putStack` @@ -290,11 +298,9 @@ export class VerkleTree { await this.saveStack(putStack) } - async del(key: Uint8Array): Promise { - const stem = key.slice(0, 31) - const suffix = key[key.length - 1] - this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}; Suffix: ${suffix}`, ['DEL']) - await this.put(key, createDeletedLeafValue()) + async del(stem: Uint8Array, suffixes: number[]): Promise { + this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}; Suffix(es): ${suffixes}`, ['DEL']) + await this.put(stem, suffixes, [createDeletedLeafValue()]) } /** * Helper method for updating or creating the parent internal node for a given leaf node diff --git a/packages/verkle/test/verkle.spec.ts b/packages/verkle/test/verkle.spec.ts index 941b9b7f3d6..f7674502604 100644 --- a/packages/verkle/test/verkle.spec.ts +++ b/packages/verkle/test/verkle.spec.ts @@ -78,7 +78,7 @@ describe('Verkle tree', () => { assert.deepEqual(res.remaining, presentKeys[0]) for (let i = 0; i < presentKeys.length; i++) { - await tree.put(presentKeys[i], values[i]) + await tree.put(presentKeys[i].slice(0, 31), [presentKeys[i][31]], [values[i]]) } for (let i = 0; i < presentKeys.length; i++) { const retrievedValue = await tree.get(presentKeys[i]) @@ -234,18 +234,34 @@ describe('Verkle tree', () => { await trie['_createRootNode']() - await trie.put(hexToBytes(keys[0]), hexToBytes(values[0])) - await trie.put(hexToBytes(keys[1]), hexToBytes(values[1])) - await trie.put(hexToBytes(keys[2]), hexToBytes(values[2])) - await trie.put(hexToBytes(keys[3]), hexToBytes(values[3])) + const keyWithMultipleValues = hexToBytes(keys[0]).slice(0, 31) + await trie.put( + keyWithMultipleValues, + [parseInt(keys[0].slice(-1)[0]), parseInt(keys[1].slice(-1)[0])], + [hexToBytes(values[0]), hexToBytes(values[1])], + ) + await trie.put( + hexToBytes(keys[2]).slice(0, 31), + [parseInt(keys[2].slice(-1)[0])], + [hexToBytes(values[2])], + ) + await trie.put( + hexToBytes(keys[3]).slice(0, 31), + [parseInt(keys[3].slice(-1)[0])], + [hexToBytes(values[3])], + ) assert.deepEqual(await trie.get(hexToBytes(keys[0])), hexToBytes(values[0])) assert.deepEqual(await trie.get(hexToBytes(keys[2])), hexToBytes(values[2])) assert.deepEqual(await trie.get(hexToBytes(keys[3])), hexToBytes(values[3])) - await trie.del(hexToBytes(keys[0])) + await trie.del(hexToBytes(keys[0]).slice(0, 31), [parseInt(keys[0].slice(-1)[0])]) assert.deepEqual(await trie.get(hexToBytes(keys[0])), new Uint8Array(32)) - await trie.put(hexToBytes(keys[0]), hexToBytes(values[0])) + await trie.put( + hexToBytes(keys[0]).slice(0, 31), + [parseInt(keys[0].slice(-1)[0])], + [hexToBytes(values[0])], + ) assert.deepEqual(await trie.get(hexToBytes(keys[0])), hexToBytes(values[0])) }) it('should put zeros in leaf node when del called with stem that was not in the trie before', async () => { @@ -260,7 +276,8 @@ describe('Verkle tree', () => { await trie['_createRootNode']() assert.deepEqual(await trie.get(hexToBytes(keys[0])), undefined) - await trie.del(hexToBytes(keys[0])) + + await trie.del(hexToBytes(keys[0]).slice(0, 31), [parseInt(keys[0].slice(-1)[0])]) const res = await trie.findPath(hexToBytes(keys[0]).slice(0, 31)) assert.ok(res.node !== null) assert.deepEqual( From 9cf5af72039fb45d0a070457390271884d0cbd6f Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:11:35 -0400 Subject: [PATCH 4/9] fix getAccount --- .../statemanager/src/statefulVerkleStateManager.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index 67bee7f2c98..d1b93603a8c 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -102,11 +102,11 @@ export class StatefulVerkleStateManager implements StateManagerInterface { const codeHash = leaf.node.getValue(VerkleLeafType.CodeHash) const codeSize = leaf.node.getValue(VerkleLeafType.CodeSize) const account = createPartialAccount({ - version: Array.isArray(version) ? bytesToInt32(version, true) : null, - balance: Array.isArray(balance) ? bytesToBigInt(balance, true) : null, - nonce: Array.isArray(nonce) ? bytesToBigInt(nonce, true) : null, - codeHash: Array.isArray(codeHash) ? codeHash : null, - codeSize: Array.isArray(codeSize) ? bytesToInt32(codeSize, true) : null, + version: version instanceof Uint8Array ? bytesToInt32(version, true) : null, + balance: balance instanceof Uint8Array ? bytesToBigInt(balance, true) : null, + nonce: nonce instanceof Uint8Array ? bytesToBigInt(nonce, true) : null, + codeHash: codeHash instanceof Uint8Array ? codeHash : null, + codeSize: codeSize instanceof Uint8Array ? bytesToInt32(codeSize, true) : null, storageRoot: null, }) // check if the account didn't exist if any of the basic keys are undefined From 1419c9dc4f77c48c83be72aa0302d55715617dd8 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:33:54 -0400 Subject: [PATCH 5/9] add todo --- packages/statemanager/src/statefulVerkleStateManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index d1b93603a8c..054381e574d 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -161,6 +161,8 @@ export class StatefulVerkleStateManager implements StateManagerInterface { account.codeHash, ], ) + } else { + // Delete account } } else { if (account !== undefined) { From bb25feac42cb9bfab08b28b5b38ecc5d37f76073 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:54:26 -0400 Subject: [PATCH 6/9] make trie.get accept suffixes --- .../src/statefulVerkleStateManager.ts | 62 ++++++------- packages/verkle/src/verkleTree.ts | 35 ++++--- packages/verkle/test/verkle.spec.ts | 91 ++++++++----------- 3 files changed, 89 insertions(+), 99 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index 054381e574d..935b36e8288 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -92,41 +92,41 @@ export class StatefulVerkleStateManager implements StateManagerInterface { const stem = getVerkleStem(this.verkleCrypto, address, 0) - // First retrieve the leaf node from the trie - const leaf = await this._trie.findPath(stem) - if (leaf.node !== null && leaf.node instanceof LeafNode) { - // We have a leaf node so let's pull out the important values for an account - const version = leaf.node.getValue(VerkleLeafType.Version) - const balance = leaf.node.getValue(VerkleLeafType.Balance) - const nonce = leaf.node.getValue(VerkleLeafType.Nonce) - const codeHash = leaf.node.getValue(VerkleLeafType.CodeHash) - const codeSize = leaf.node.getValue(VerkleLeafType.CodeSize) - const account = createPartialAccount({ - version: version instanceof Uint8Array ? bytesToInt32(version, true) : null, - balance: balance instanceof Uint8Array ? bytesToBigInt(balance, true) : null, - nonce: nonce instanceof Uint8Array ? bytesToBigInt(nonce, true) : null, - codeHash: codeHash instanceof Uint8Array ? codeHash : null, - codeSize: codeSize instanceof Uint8Array ? bytesToInt32(codeSize, true) : null, - storageRoot: null, - }) - // check if the account didn't exist if any of the basic keys are undefined - if ( - version === undefined || - balance === undefined || - nonce === undefined || - codeHash === undefined - ) { - if (this.DEBUG) { - this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) - } - this._caches?.account?.put(address, account) - } + // First retrieve the account "header" values from the trie + const accountValues = await this._trie.get(stem, [ + VerkleLeafType.Version, + VerkleLeafType.Balance, + VerkleLeafType.Nonce, + VerkleLeafType.CodeHash, + ]) + const account = createPartialAccount({ + version: accountValues[0] instanceof Uint8Array ? bytesToInt32(accountValues[0], true) : null, + balance: + accountValues[1] instanceof Uint8Array ? bytesToBigInt(accountValues[1], true) : null, + nonce: accountValues[2] instanceof Uint8Array ? bytesToBigInt(accountValues[2], true) : null, + codeHash: accountValues[3] instanceof Uint8Array ? accountValues[3] : null, + codeSize: + accountValues[4] instanceof Uint8Array ? bytesToInt32(accountValues[4], true) : null, + storageRoot: null, + }) + // check if the account didn't exist if any of the basic keys are undefined + if ( + account.version === null || + account.balance === null || + account.nonce === null || + account.codeHash === null + ) { if (this.DEBUG) { - this._debug(`getAccount address=${address.toString()} stem=${short(stem)}`) + this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) } - return account + this._caches?.account?.put(address, account) + } + + if (this.DEBUG) { + this._debug(`getAccount address=${address.toString()} stem=${short(stem)}`) } + return account } putAccount = async (address: Address, account?: Account): Promise => { diff --git a/packages/verkle/src/verkleTree.ts b/packages/verkle/src/verkleTree.ts index 8c6ca6f015d..6547ca1c398 100644 --- a/packages/verkle/src/verkleTree.ts +++ b/packages/verkle/src/verkleTree.ts @@ -152,26 +152,33 @@ export class VerkleTree { } /** - * Gets a value given a `key` - * @param key - the key to search for - * @returns A Promise that resolves to `Uint8Array` if a value was found or `undefined` if no value was found. + * Gets values at a given verkle `stem` and set of suffixes + * @param stem - the stem of the leaf node where we're seaking values + * @param suffixes - an array of suffixes corresponding to the values desired + * @returns A Promise that resolves to an array of `Uint8Array`s if a value + * was found or `undefined` if no value was found at a given suffixes. */ - async get(key: Uint8Array): Promise { - if (key.length !== 32) throw new Error(`expected key with length 32; got ${key.length}`) - const stem = key.slice(0, 31) - const suffix = key[key.length - 1] - this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}; Suffix: ${suffix}`, ['GET']) + async get(stem: Uint8Array, suffixes: number[]): Promise<(Uint8Array | undefined)[]> { + if (stem.length !== 31) throw new Error(`expected stem with length 31; got ${stem.length}`) + this.DEBUG && this.debug(`Stem: ${bytesToHex(stem)}; Suffix: ${suffixes}`, ['GET']) const res = await this.findPath(stem) if (res.node instanceof LeafNode) { // The retrieved leaf node contains an array of 256 possible values. - // The index of the value we want is at the key's last byte - const value = res.node.getValue(suffix) - this.DEBUG && - this.debug(`Value: ${value === undefined ? 'undefined' : bytesToHex(value)}`, ['GET']) - return value + // We read all the suffixes to get the desired values + const values = [] + for (const suffix of suffixes) { + const value = res.node.getValue(suffix) + this.DEBUG && + this.debug( + `Suffix: ${suffix}; Value: ${value === undefined ? 'undefined' : bytesToHex(value)}`, + ['GET'], + ) + values.push(value) + } + return values } - return + return [] } /** diff --git a/packages/verkle/test/verkle.spec.ts b/packages/verkle/test/verkle.spec.ts index f7674502604..405fb465439 100644 --- a/packages/verkle/test/verkle.spec.ts +++ b/packages/verkle/test/verkle.spec.ts @@ -80,12 +80,16 @@ describe('Verkle tree', () => { for (let i = 0; i < presentKeys.length; i++) { await tree.put(presentKeys[i].slice(0, 31), [presentKeys[i][31]], [values[i]]) } - for (let i = 0; i < presentKeys.length; i++) { - const retrievedValue = await tree.get(presentKeys[i]) + const stem = presentKeys[0].slice(0, 31) + const retrievedValues = await tree.get(stem, [presentKeys[0][31], presentKeys[1][31]]) + assert.deepEqual(retrievedValues[0], values[0]) + assert.deepEqual(retrievedValues[1], values[1]) + for (let i = 2; i < presentKeys.length; i++) { + const retrievedValue = await tree.get(presentKeys[i].slice(0, 31), [presentKeys[i][31]]) if (retrievedValue === undefined) { assert.fail('Value not found') } - assert.ok(equalsBytes(retrievedValue, values[i])) + assert.ok(equalsBytes(retrievedValue[0]!, values[i])) } // Verify that findPath returns a path that demonstrates the nonexistence of a key @@ -109,13 +113,13 @@ describe('Verkle tree', () => { '0x318dfa512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', // A key with a partially matching stem 0x318dfa51 to above key '0x318dfa513b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', - ] as PrefixedHexString[] + ].map((key) => hexToBytes(key as PrefixedHexString)) const values = [ '0x320122e8584be00d000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000001', '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0300000000000000000000000000000000000000000000000000000000000000', - ] as PrefixedHexString[] + ].map((key) => hexToBytes(key as PrefixedHexString)) const trie = await createVerkleTree({ verkleCrypto, db: new MapDB(), @@ -124,12 +128,12 @@ describe('Verkle tree', () => { await trie['_createRootNode']() let putStack: [Uint8Array, VerkleNode][] = [] - const stem1 = hexToBytes(keys[0]).slice(0, 31) + const stem1 = keys[0].slice(0, 31) // Create first leaf node const leafNode1 = await LeafNode.create(stem1, verkleCrypto) - leafNode1.setValue(hexToBytes(keys[0])[31], hexToBytes(values[0])) - leafNode1.setValue(hexToBytes(keys[1])[31], hexToBytes(values[1])) + leafNode1.setValue(keys[0][31], values[0]) + leafNode1.setValue(keys[1][31], values[1]) putStack.push([leafNode1.hash(), leafNode1]) @@ -147,12 +151,12 @@ describe('Verkle tree', () => { assert.deepEqual(res.node?.commitment, leafNode1.commitment) // Retrieve a value from the leaf node - const val1 = await trie.get(hexToBytes(keys[1])) - assert.deepEqual(val1, hexToBytes(values[1])) + const val1 = await trie.get(keys[1].slice(0, 31), [keys[1][31]]) + assert.deepEqual(val1[0], values[1]) // Put a second leaf node in the tree with a partially matching stem putStack = [] - const stem2 = hexToBytes(keys[2]).slice(0, 31) + const stem2 = keys[2].slice(0, 31) // Find path to closest node in tree const foundPath = await trie.findPath(stem2) @@ -162,13 +166,13 @@ describe('Verkle tree', () => { // Create new leaf node const leafNode2 = await LeafNode.create(stem2, verkleCrypto) - leafNode2.setValue(hexToBytes(keys[2])[31], hexToBytes(values[2])) + leafNode2.setValue(keys[2][31], values[2]) putStack.push([leafNode2.hash(), leafNode2]) const nearestNode = foundPath.stack.pop()![0] // Verify that another leaf node is "nearest" node assert.equal(nearestNode.type, VerkleNodeType.Leaf) - assert.deepEqual((nearestNode as LeafNode).getValue(2), hexToBytes(values[1])) + assert.deepEqual((nearestNode as LeafNode).getValue(2), values[1]) // Compute the portion of stem1 and stem2 that match (i.e. the partial path closest to stem2) const partialMatchingStemIndex = matchingBytesLength(stem1, stem2) @@ -207,8 +211,8 @@ describe('Verkle tree', () => { res2 = await trie.findPath(stem2) assert.equal(res2.remaining.length, 0, 'confirm full path was found') assert.equal(res2.stack.length, 2, 'confirm node is at depth 2') - const val2 = await trie.get(hexToBytes(keys[2])) - assert.deepEqual(val2, hexToBytes(values[2]), 'confirm values[2] can be retrieved from trie') + const val2 = await trie.get(keys[2].slice(0, 31), [keys[2][31]]) + assert.deepEqual(val2[0], values[2], 'confirm values[2] can be retrieved from trie') }) it('should sequentially put->find->delete->put values', async () => { @@ -220,13 +224,13 @@ describe('Verkle tree', () => { '0x318dfa512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', // A key with a partially matching stem 0x318dfa51 to above key '0x318dfa513b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d02', - ] as PrefixedHexString[] + ].map((key) => hexToBytes(key as PrefixedHexString)) const values = [ '0x320122e8584be00d000000000000000000000000000000000000000000000000', '0x0000000000000000000000000000000000000000000000000000000000000001', '0x0000000000000000000000000000000000000000000000000000000000000000', '0x0300000000000000000000000000000000000000000000000000000000000000', - ] as PrefixedHexString[] + ].map((key) => hexToBytes(key as PrefixedHexString)) const trie = await createVerkleTree({ verkleCrypto, db: new MapDB(), @@ -234,40 +238,22 @@ describe('Verkle tree', () => { await trie['_createRootNode']() - const keyWithMultipleValues = hexToBytes(keys[0]).slice(0, 31) - await trie.put( - keyWithMultipleValues, - [parseInt(keys[0].slice(-1)[0]), parseInt(keys[1].slice(-1)[0])], - [hexToBytes(values[0]), hexToBytes(values[1])], - ) - await trie.put( - hexToBytes(keys[2]).slice(0, 31), - [parseInt(keys[2].slice(-1)[0])], - [hexToBytes(values[2])], - ) - await trie.put( - hexToBytes(keys[3]).slice(0, 31), - [parseInt(keys[3].slice(-1)[0])], - [hexToBytes(values[3])], - ) - assert.deepEqual(await trie.get(hexToBytes(keys[0])), hexToBytes(values[0])) - assert.deepEqual(await trie.get(hexToBytes(keys[2])), hexToBytes(values[2])) - assert.deepEqual(await trie.get(hexToBytes(keys[3])), hexToBytes(values[3])) + const keyWithMultipleValues = keys[0].slice(0, 31) + await trie.put(keyWithMultipleValues, [keys[0][31], keys[1][31]], [values[0], values[1]]) + await trie.put(keys[2].slice(0, 31), [keys[2][31]], [values[2]]) + await trie.put(keys[3].slice(0, 31), [keys[3][31]], [values[3]]) + assert.deepEqual((await trie.get(keys[0].slice(0, 31), [keys[0][31]]))[0], values[0]) + assert.deepEqual((await trie.get(keys[2].slice(0, 31), [keys[2][31]]))[0], values[2]) + assert.deepEqual((await trie.get(keys[3].slice(0, 31), [keys[3][31]]))[0], values[3]) - await trie.del(hexToBytes(keys[0]).slice(0, 31), [parseInt(keys[0].slice(-1)[0])]) - assert.deepEqual(await trie.get(hexToBytes(keys[0])), new Uint8Array(32)) + await trie.del(keys[0].slice(0, 31), [keys[0][31]]) + assert.deepEqual((await trie.get(keys[0].slice(0, 31), [keys[0][31]]))[0], new Uint8Array(32)) - await trie.put( - hexToBytes(keys[0]).slice(0, 31), - [parseInt(keys[0].slice(-1)[0])], - [hexToBytes(values[0])], - ) - assert.deepEqual(await trie.get(hexToBytes(keys[0])), hexToBytes(values[0])) + await trie.put(keys[0].slice(0, 31), [keys[0][31]], [values[0]]) + assert.deepEqual((await trie.get(keys[0].slice(0, 31), [keys[0][31]]))[0], values[0]) }) it('should put zeros in leaf node when del called with stem that was not in the trie before', async () => { - const keys = [ - '0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01', - ] as PrefixedHexString[] + const keys = [hexToBytes('0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01')] const trie = await createVerkleTree({ verkleCrypto, @@ -275,14 +261,11 @@ describe('Verkle tree', () => { }) await trie['_createRootNode']() - assert.deepEqual(await trie.get(hexToBytes(keys[0])), undefined) + assert.deepEqual(await trie.get(keys[0].slice(0, 31), [keys[0][31]]), []) - await trie.del(hexToBytes(keys[0]).slice(0, 31), [parseInt(keys[0].slice(-1)[0])]) - const res = await trie.findPath(hexToBytes(keys[0]).slice(0, 31)) + await trie.del(keys[0].slice(0, 31), [keys[0][31]]) + const res = await trie.findPath(keys[0].slice(0, 31)) assert.ok(res.node !== null) - assert.deepEqual( - (res.node as LeafNode).values[hexToBytes(keys[0])[31]], - VerkleLeafNodeValue.Deleted, - ) + assert.deepEqual((res.node as LeafNode).values[keys[0][31]], VerkleLeafNodeValue.Deleted) }) }) From 8c0605cd7899bd749d91b79e2c9aa167f8c400ac Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:59:57 -0400 Subject: [PATCH 7/9] clean up reference --- .../src/statefulVerkleStateManager.ts | 69 +++++++------------ 1 file changed, 26 insertions(+), 43 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index 935b36e8288..2e814872556 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -9,17 +9,13 @@ import { bytesToInt32, createAccountFromRLP, createPartialAccount, - getVerkleKey, + decodeVerkleLeafBasicData, + encodeVerkleLeafBasicData, getVerkleStem, setLengthRight, short, } from '@ethereumjs/util' -import { - LeafNode, - VerkleLeafNodeValue, - VerkleTree, - createUntouchedLeafValue, -} from '@ethereumjs/verkle' +import { VerkleTree, createUntouchedLeafValue } from '@ethereumjs/verkle' import debugDefault from 'debug' import { keccak256 } from 'ethereum-cryptography/keccak.js' @@ -94,29 +90,24 @@ export class StatefulVerkleStateManager implements StateManagerInterface { // First retrieve the account "header" values from the trie const accountValues = await this._trie.get(stem, [ - VerkleLeafType.Version, - VerkleLeafType.Balance, - VerkleLeafType.Nonce, + VerkleLeafType.BasicData, VerkleLeafType.CodeHash, ]) - const account = createPartialAccount({ - version: accountValues[0] instanceof Uint8Array ? bytesToInt32(accountValues[0], true) : null, - balance: - accountValues[1] instanceof Uint8Array ? bytesToBigInt(accountValues[1], true) : null, - nonce: accountValues[2] instanceof Uint8Array ? bytesToBigInt(accountValues[2], true) : null, - codeHash: accountValues[3] instanceof Uint8Array ? accountValues[3] : null, - codeSize: - accountValues[4] instanceof Uint8Array ? bytesToInt32(accountValues[4], true) : null, - storageRoot: null, - }) + let account + if (accountValues[0] !== undefined) { + const basicData = decodeVerkleLeafBasicData(accountValues[0]!) + account = createPartialAccount({ + version: basicData.version, + balance: basicData.balance, + nonce: basicData.nonce, + codeHash: accountValues[1] instanceof Uint8Array ? accountValues[1] : null, + codeSize: basicData.codeSize, + storageRoot: null, // TODO: Add storage stuff + }) + } // check if the account didn't exist if any of the basic keys are undefined - if ( - account.version === null || - account.balance === null || - account.nonce === null || - account.codeHash === null - ) { + else if (accountValues[0] === undefined || accountValues[1] === undefined) { if (this.DEBUG) { this._debug(`getAccount address=${address.toString()} from DB (non-existent)`) } @@ -142,24 +133,16 @@ export class StatefulVerkleStateManager implements StateManagerInterface { if (this._caches?.account === undefined) { if (account !== undefined) { const stem = getVerkleStem(this.verkleCrypto, address, 0) + const basicDataBytes = encodeVerkleLeafBasicData({ + version: account.version, + balance: account.balance, + nonce: account.nonce, + codeSize: account.codeSize, + }) await this._trie.put( stem, - [ - VerkleLeafType.Version, - VerkleLeafType.Balance, - VerkleLeafType.Nonce, - VerkleLeafType.CodeHash, - ], - [ - account.balance !== null - ? setLengthRight(bigIntToBytes(account.balance, true), 32) - : createUntouchedLeafValue(), - account.balance !== null - ? setLengthRight(bigIntToBytes(account.balance, true), 32) - : createUntouchedLeafValue(), - setLengthRight(bigIntToBytes(account.nonce, true), 32), - account.codeHash, - ], + [VerkleLeafType.BasicData, VerkleLeafType.CodeHash], + [basicDataBytes, account.codeHash], ) } else { // Delete account @@ -238,7 +221,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface { throw new Error('Method not implemented.') } - verifyVerkleProof?(stateRoot: Uint8Array): boolean { + verifyVerkleProof?(): boolean { throw new Error('Method not implemented.') } verifyPostState?(): boolean { From ef37d62255ba0e4b9fc99314bb8f0bf9fc0d4b6d Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Sat, 31 Aug 2024 13:19:26 -0400 Subject: [PATCH 8/9] Add reserved bytes to encodeBasicData function --- .../src/statefulVerkleStateManager.ts | 45 +++++++++---------- packages/util/src/verkle.ts | 14 ++++-- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index 2e814872556..8ace9149ea6 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -129,30 +129,29 @@ export class StatefulVerkleStateManager implements StateManagerInterface { account && account.isEmpty() ? 'yes' : 'no' }`, ) - - if (this._caches?.account === undefined) { - if (account !== undefined) { - const stem = getVerkleStem(this.verkleCrypto, address, 0) - const basicDataBytes = encodeVerkleLeafBasicData({ - version: account.version, - balance: account.balance, - nonce: account.nonce, - codeSize: account.codeSize, - }) - await this._trie.put( - stem, - [VerkleLeafType.BasicData, VerkleLeafType.CodeHash], - [basicDataBytes, account.codeHash], - ) - } else { - // Delete account - } + } + if (this._caches?.account === undefined) { + if (account !== undefined) { + const stem = getVerkleStem(this.verkleCrypto, address, 0) + const basicDataBytes = encodeVerkleLeafBasicData({ + version: account.version, + balance: account.balance, + nonce: account.nonce, + codeSize: account.codeSize, + }) + await this._trie.put( + stem, + [VerkleLeafType.BasicData, VerkleLeafType.CodeHash], + [basicDataBytes, account.codeHash], + ) + } else { + // Delete account + } + } else { + if (account !== undefined) { + this._caches?.account?.put(address, account, true) } else { - if (account !== undefined) { - this._caches?.account?.put(address, account, true) - } else { - this._caches?.account?.del(address) - } + this._caches?.account?.del(address) } } } diff --git a/packages/util/src/verkle.ts b/packages/util/src/verkle.ts index 28392d68cd8..ed54f666352 100644 --- a/packages/util/src/verkle.ts +++ b/packages/util/src/verkle.ts @@ -247,12 +247,18 @@ export function decodeVerkleLeafBasicData(encodedBasicData: Uint8Array): VerkleL } export function encodeVerkleLeafBasicData(basicData: VerkleLeafBasicData): Uint8Array { - const encodedVersion = setLengthLeft(int32ToBytes(basicData.version), VERKLE_VERSION_BYTES_LENGTH) + const encodedVersion = setLengthRight( + int32ToBytes(basicData.version, true), + VERKLE_VERSION_BYTES_LENGTH, + ) // Per EIP-6800, bytes 1-4 are reserved for future use const reservedBytes = new Uint8Array([0, 0, 0]) - const encodedNonce = setLengthLeft(bigIntToBytes(basicData.nonce), VERKLE_NONCE_BYTES_LENGTH) - const encodedCodeSize = setLengthLeft( - int32ToBytes(basicData.codeSize), + const encodedNonce = setLengthRight( + bigIntToBytes(basicData.nonce, true), + VERKLE_NONCE_BYTES_LENGTH, + ) + const encodedCodeSize = setLengthRight( + int32ToBytes(basicData.codeSize, true), VERKLE_CODE_SIZE_BYTES_LENGTH, ) const encodedBalance = setLengthLeft( From adad63130199aa81bb9e08af9f5e30eb6a47e1ad Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Tue, 3 Sep 2024 06:27:07 -0400 Subject: [PATCH 9/9] FIx encoding again --- .../src/statefulVerkleStateManager.ts | 8 +++---- packages/util/src/verkle.ts | 22 +++++++------------ packages/util/test/verkle.spec.ts | 19 ++++++++++++++++ 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/packages/statemanager/src/statefulVerkleStateManager.ts b/packages/statemanager/src/statefulVerkleStateManager.ts index 8ace9149ea6..95c755d2a40 100644 --- a/packages/statemanager/src/statefulVerkleStateManager.ts +++ b/packages/statemanager/src/statefulVerkleStateManager.ts @@ -4,18 +4,14 @@ import { type Address, MapDB, VerkleLeafType, - bigIntToBytes, - bytesToBigInt, - bytesToInt32, createAccountFromRLP, createPartialAccount, decodeVerkleLeafBasicData, encodeVerkleLeafBasicData, getVerkleStem, - setLengthRight, short, } from '@ethereumjs/util' -import { VerkleTree, createUntouchedLeafValue } from '@ethereumjs/verkle' +import { VerkleTree } from '@ethereumjs/verkle' import debugDefault from 'debug' import { keccak256 } from 'ethereum-cryptography/keccak.js' @@ -96,6 +92,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface { let account if (accountValues[0] !== undefined) { + console.log(accountValues) const basicData = decodeVerkleLeafBasicData(accountValues[0]!) account = createPartialAccount({ version: basicData.version, @@ -139,6 +136,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface { nonce: account.nonce, codeSize: account.codeSize, }) + console.log(basicDataBytes) await this._trie.put( stem, [VerkleLeafType.BasicData, VerkleLeafType.CodeHash], diff --git a/packages/util/src/verkle.ts b/packages/util/src/verkle.ts index ed54f666352..f34aae10107 100644 --- a/packages/util/src/verkle.ts +++ b/packages/util/src/verkle.ts @@ -238,27 +238,21 @@ export function decodeVerkleLeafBasicData(encodedBasicData: Uint8Array): VerkleL VERKLE_BALANCE_OFFSET + VERKLE_BALANCE_BYTES_LENGTH, ) - const version = bytesToInt32(versionBytes, true) - const nonce = bytesToBigInt(nonceBytes, true) - const codeSize = bytesToInt32(codeSizeBytes, true) - const balance = bytesToBigInt(balanceBytes, true) + const version = bytesToInt32(versionBytes) + const nonce = bytesToBigInt(nonceBytes) + const codeSize = bytesToInt32(codeSizeBytes) + const balance = bytesToBigInt(balanceBytes) return { version, nonce, codeSize, balance } } export function encodeVerkleLeafBasicData(basicData: VerkleLeafBasicData): Uint8Array { - const encodedVersion = setLengthRight( - int32ToBytes(basicData.version, true), - VERKLE_VERSION_BYTES_LENGTH, - ) + const encodedVersion = setLengthLeft(int32ToBytes(basicData.version), VERKLE_VERSION_BYTES_LENGTH) // Per EIP-6800, bytes 1-4 are reserved for future use const reservedBytes = new Uint8Array([0, 0, 0]) - const encodedNonce = setLengthRight( - bigIntToBytes(basicData.nonce, true), - VERKLE_NONCE_BYTES_LENGTH, - ) - const encodedCodeSize = setLengthRight( - int32ToBytes(basicData.codeSize, true), + const encodedNonce = setLengthLeft(bigIntToBytes(basicData.nonce), VERKLE_NONCE_BYTES_LENGTH) + const encodedCodeSize = setLengthLeft( + int32ToBytes(basicData.codeSize), VERKLE_CODE_SIZE_BYTES_LENGTH, ) const encodedBalance = setLengthLeft( diff --git a/packages/util/test/verkle.spec.ts b/packages/util/test/verkle.spec.ts index d8aa627ff22..ff32c119c36 100644 --- a/packages/util/test/verkle.spec.ts +++ b/packages/util/test/verkle.spec.ts @@ -3,12 +3,16 @@ import { assert, beforeAll, describe, it } from 'vitest' import * as verkleBlockJSON from '../../statemanager/test/testdata/verkleKaustinen6Block72.json' import { + Account, type VerkleCrypto, type VerkleExecutionWitness, VerkleLeafType, bytesToHex, concatBytes, + createAddressFromPrivateKey, createAddressFromString, + decodeVerkleLeafBasicData, + encodeVerkleLeafBasicData, getVerkleKey, getVerkleStem, hexToBytes, @@ -81,3 +85,18 @@ describe('should generate valid tree keys', () => { } }) }) + +describe('should encode and decode basic data values', () => { + const account = new Account(2n, 123n) + it('should encode basicData to 32 bytes', () => { + const basicDataBytes = encodeVerkleLeafBasicData(account) + assert.equal(basicDataBytes.length, 32) + assert.equal( + basicDataBytes.slice(4, 12)[7], + 2, + 'confirm that last byte of nonce slice is equal to nonce (i.e. coded as bigEndian', + ) + const decodedData = decodeVerkleLeafBasicData(basicDataBytes) + assert.equal(decodedData.balance, 123n) + }) +})