vm: adjust test runner with verkle state management

packages/common/src/chains.ts

@@ -117,6 +117,10 @@ export const Mainnet: ChainConfig = {
       name: 'prague',
       block: null,
     },
+    {
+      name: 'verkle',
+      block: null,
+    },
   ],
   bootstrapNodes: [
     {

packages/common/src/enums.ts

@@ -72,6 +72,7 @@ export enum Hardfork {
   Cancun = 'cancun',
   Prague = 'prague',
   Osaka = 'osaka',
+  Verkle = 'verkle',
 }
 
 export enum ConsensusType {

packages/common/src/hardforks.ts

@@ -170,4 +170,8 @@ export const hardforksDict: HardforksDict = {
   osaka: {
     eips: [2935, 6800],
   },
+  // TODO: WIP
+  verkle: {
+    eips: [2935, 6800],
+  },
 }

packages/evm/src/evm.ts

@@ -85,6 +85,7 @@ export class EVM implements EVMInterface {
     Hardfork.Cancun,
     Hardfork.Prague,
     Hardfork.Osaka,
+    Hardfork.Verkle,
   ]
   protected _tx?: {
     gasPrice: bigint

packages/statemanager/src/statefulVerkleStateManager.ts

@@ -147,7 +147,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
   putAccount = async (address: Address, account?: Account): Promise<void> => {
     if (this.DEBUG) {
       this._debug(
-        `Save account address=${address} nonce=${account?.nonce} balance=${
+        `putAccount address=${address} nonce=${account?.nonce} balance=${
           account?.balance
         } contract=${account && account.isContract() ? 'yes' : 'no'} empty=${
           account && account.isEmpty() ? 'yes' : 'no'
@@ -446,10 +446,12 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
   }
 
   getStateRoot(): Promise<Uint8Array> {
-    throw new Error('Method not implemented.')
+    return Promise.resolve(this._trie.root())
   }
-  setStateRoot(_stateRoot: Uint8Array, _clearCache?: boolean): Promise<void> {
-    throw new Error('Method not implemented.')
+  setStateRoot(stateRoot: Uint8Array, clearCache?: boolean): Promise<void> {
+    this._trie.root(stateRoot)
+    clearCache === true && this.clearCaches()
+    return Promise.resolve()
   }
   hasStateRoot(_root: Uint8Array): Promise<boolean> {
     throw new Error('Method not implemented.')
@@ -461,7 +463,7 @@ export class StatefulVerkleStateManager implements StateManagerInterface {
     throw new Error('Method not implemented.')
   }
   clearCaches(): void {
-    throw new Error('Method not implemented.')
+    this._caches?.clear()
   }
   shallowCopy(_downlevelCaches?: boolean): StateManagerInterface {
     throw new Error('Method not implemented.')

packages/statemanager/src/util.ts

@@ -1,4 +1,4 @@
-import { Account } from '@ethereumjs/util'
+import { Account, bytesToHex } from '@ethereumjs/util'
 
 import type { AccountFields, StateManagerInterface } from '@ethereumjs/common'
 import type { Address } from '@ethereumjs/util'
@@ -15,5 +15,14 @@ export async function modifyAccountFields(
   account.storageRoot = accountFields.storageRoot ?? account.storageRoot
   account.codeHash = accountFields.codeHash ?? account.codeHash
   account.codeSize = accountFields.codeSize ?? account.codeSize
+  // @ts-ignore
+  if (stateManager['_debug'] !== undefined) {
+    for (const [field, value] of Object.entries(accountFields)) {
+      //@ts-ignore
+      stateManager['_debug'](
+        `modifyAccountFields address=${address.toString()} ${field}=${value instanceof Uint8Array ? bytesToHex(value) : value} `,
+      )
+    }
+  }
   await stateManager.putAccount(address, account)
 }

packages/vm/package.json

@@ -78,6 +78,7 @@
   "devDependencies": {
     "@ethereumjs/blockchain": "^7.3.0",
     "@ethereumjs/ethash": "^3.0.3",
+    "@ethereumjs/verkle": "0.1.0",
     "@paulmillr/trusted-setups": "^0.1.2",
     "@types/benchmark": "^1.0.33",
     "@types/core-js": "^2.5.0",

packages/vm/test/tester/config.ts

@@ -108,6 +108,7 @@ const normalHardforks = [
   'arrowGlacier', // This network has no tests, but need to add it due to common generation logic
   'cancun',
   'prague',
+  'verkle',
 ]
 
 const transitionNetworks = {
@@ -277,6 +278,69 @@ function setupCommonWithNetworks(network: string, ttd?: number, timestamp?: numb
   return common
 }
 
+/**
+ * Returns a common instance configured for verkle
+ * @param network Network target (this can include EIPs, such as Byzantium+2537+2929)
+ * @param ttd If set: total terminal difficulty to switch to merge
+ * @returns
+ */
+function setupCommonForVerkle(network: string, timestamp?: number, kzg?: KZG) {
+  let ttd
+  // hard fork that verkle tests are filled on
+  const hfName = 'shanghai'
+  const mainnetCommon = new Common({ chain: Mainnet, hardfork: hfName })
+  const hardforks = mainnetCommon.hardforks().slice(0, 17) // skip hardforks after Shanghai
+  const testHardforks: HardforkTransitionConfig[] = []
+  for (const hf of hardforks) {
+    // check if we enable this hf
+    // disable dao hf by default (if enabled at block 0 forces the first 10 blocks to have dao-hard-fork in extraData of block header)
+    if (mainnetCommon.gteHardfork(hf.name) && hf.name !== Hardfork.Dao) {
+      // this hardfork should be activated at block 0
+      testHardforks.push({
+        name: hf.name,
+        // Current type definition Partial<Chain> in Common is currently not allowing to pass in forkHash
+        // forkHash: hf.forkHash,
+        block: 0,
+      })
+    } else {
+      // disable hardforks newer than the test hardfork (but do add "support" for it, it just never gets activated)
+      if (
+        (ttd === undefined && timestamp === undefined) ||
+        (hf.name === 'paris' && ttd !== undefined)
+      ) {
+        testHardforks.push({
+          name: hf.name,
+          block: null,
+        })
+      }
+      if (timestamp !== undefined && hf.name !== Hardfork.Dao) {
+        testHardforks.push({
+          name: hf.name,
+          block: null,
+          timestamp,
+        })
+      }
+    }
+  }
+
+  testHardforks.push({ name: 'verkle', block: null })
+
+  const common = createCustomCommon(
+    {
+      hardforks: testHardforks,
+      defaultHardfork: 'verkle',
+    },
+    Mainnet,
+    { eips: [3607], customCrypto: { kzg } },
+  )
+  // Activate EIPs
+  const eips = network.match(/(?<=\+)(.\d+)/g)
+  if (eips) {
+    common.setEIPs(eips.map((e: string) => parseInt(e)))
+  }
+  return common
+}
+
 /**
  * Returns a Common for the given network (a test parameter)
  * @param network - the network field of a test.
@@ -287,6 +351,8 @@ function setupCommonWithNetworks(network: string, ttd?: number, timestamp?: numb
  * @returns the Common which should be used
  */
 export function getCommon(network: string, kzg?: KZG): Common {
+  // Special handler for verkle tests
+  if (network.toLowerCase().includes('verkle')) return setupCommonForVerkle(network, undefined, kzg)
   if (retestethAlias[network as keyof typeof retestethAlias] !== undefined) {
     network = retestethAlias[network as keyof typeof retestethAlias]
   }

packages/vm/test/tester/index.ts

@@ -141,6 +141,7 @@ async function runTests() {
     profile: boolean
     bls: EVMBLSInterface
     bn254: EVMBN254Interface
+    stateManager: string
   } = {
     forkConfigVM: FORK_CONFIG_VM,
     forkConfigTestSuite: FORK_CONFIG_TEST_SUITE,
@@ -155,6 +156,7 @@ async function runTests() {
     bls,
     profile: RUN_PROFILER,
     bn254,
+    stateManager: argv.stateManager,
   }
 
   /**

packages/vm/test/tester/runners/BlockchainTestsRunner.ts

@@ -4,7 +4,7 @@ import { ConsensusAlgorithm } from '@ethereumjs/common'
 import { Ethash } from '@ethereumjs/ethash'
 import { MerklePatriciaTrie } from '@ethereumjs/mpt'
 import { RLP } from '@ethereumjs/rlp'
-import { Caches, MerkleStateManager } from '@ethereumjs/statemanager'
+import { Caches, MerkleStateManager, StatefulVerkleStateManager } from '@ethereumjs/statemanager'
 import { createTxFromRLP } from '@ethereumjs/tx'
 import {
   MapDB,
@@ -15,14 +15,17 @@ import {
   stripHexPrefix,
   toBytes,
 } from '@ethereumjs/util'
+import { createVerkleTree } from '@ethereumjs/verkle'
+import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 
 import { buildBlock, createVM, runBlock } from '../../../src/index.js'
 import { setupPreConditions, verifyPostConditions } from '../../util.js'
 
 import type { Block } from '@ethereumjs/block'
 import type { Blockchain, ConsensusDict } from '@ethereumjs/blockchain'
-import type { Common } from '@ethereumjs/common'
+import type { Common, StateManagerInterface } from '@ethereumjs/common'
 import type { PrefixedHexString } from '@ethereumjs/util'
+import type { VerkleTree } from '@ethereumjs/verkle'
 import type * as tape from 'tape'
 
 function formatBlockHeader(data: any) {
@@ -35,7 +38,7 @@ function formatBlockHeader(data: any) {
 
 export async function runBlockchainTest(options: any, testData: any, t: tape.Test) {
   // ensure that the test data is the right fork data
-  if (testData.network !== options.forkConfigTestSuite) {
+  if (testData.network.toLowerCase() !== options.forkConfigTestSuite) {
     t.comment(`skipping test: no data available for ${options.forkConfigTestSuite}`)
     return
   }
@@ -46,13 +49,26 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes
   let common = options.common.copy() as Common
   common.setHardforkBy({ blockNumber: 0 })
 
+  let stateTree: MerklePatriciaTrie | VerkleTree
+  let stateManager: StateManagerInterface
+
+  if (options.stateManager === 'verkle') {
+    const verkleCrypto = await loadVerkleCrypto()
+    stateTree = await createVerkleTree({ verkleCrypto, db: new MapDB() })
+    stateManager = new StatefulVerkleStateManager({
+      verkleCrypto,
+      trie: stateTree,
+    })
+  } else {
+    stateTree = new MerklePatriciaTrie({ useKeyHashing: true, common })
+    stateManager = new MerkleStateManager({
+      caches: new Caches(),
+      trie: stateTree,
+      common,
+    })
+  }
+
   let cacheDB = new MapDB()
-  let state = new MerklePatriciaTrie({ useKeyHashing: true, common })
-  let stateManager = new MerkleStateManager({
-    caches: new Caches(),
-    trie: state,
-    common,
-  })
 
   let validatePow = false
   // Only run with block validation when sealEngine present in test file
@@ -86,7 +102,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes
   })
 
   if (validatePow) {
-    ;(blockchain.consensus as EthashConsensus)._ethash!.cacheDB = cacheDB as any
+    ;(blockchain.consensus as EthashConsensus)._ethash!.cacheDB = cacheDB
   }
 
   const begin = Date.now()
@@ -224,7 +240,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes
           const headBlock = await (vm.blockchain as Blockchain).getIteratorHead()
           await vm.stateManager.setStateRoot(headBlock.header.stateRoot)
         } else {
-          await verifyPostConditions(state, testData.postState, t)
+          await verifyPostConditions(stateTree, testData.postState, t)
         }
 
         throw e
@@ -242,7 +258,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes
   }
 
   t.equal(
-    bytesToHex((blockchain as any)._headHeaderHash),
+    bytesToHex(blockchain['_headHeaderHash']),
     '0x' + testData.lastblockhash,
     'correct last header block',
   )
@@ -251,6 +267,5 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes
   const timeSpent = `${(end - begin) / 1000} secs`
   t.comment(`Time: ${timeSpent}`)
 
-  // @ts-ignore Explicitly delete objects for memory optimization (early GC)
-  common = blockchain = state = stateManager = vm = cacheDB = null // eslint-disable-line
+  common = blockchain = stateTree = stateManager = vm = cacheDB = null as any
 }

packages/vm/test/tester/runners/GeneralStateTestsRunner.ts

@@ -2,18 +2,23 @@ import { Block } from '@ethereumjs/block'
 import { createBlockchain } from '@ethereumjs/blockchain'
 import { type InterpreterStep } from '@ethereumjs/evm'
 import { MerklePatriciaTrie } from '@ethereumjs/mpt'
-import { Caches, MerkleStateManager } from '@ethereumjs/statemanager'
+import { Caches, MerkleStateManager, StatefulVerkleStateManager } from '@ethereumjs/statemanager'
 import {
   Account,
+  MapDB,
   bytesToHex,
   createAddressFromString,
   equalsBytes,
   toBytes,
 } from '@ethereumjs/util'
+import { createVerkleTree } from '@ethereumjs/verkle'
+import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 
 import { createVM, runTx } from '../../../src/index.js'
 import { makeBlockFromEnv, makeTx, setupPreConditions } from '../../util.js'
 
+import type { StateManagerInterface } from '@ethereumjs/common'
+import type { VerkleTree } from '@ethereumjs/verkle'
 import type * as tape from 'tape'
 
 function parseTestCases(
@@ -74,23 +79,34 @@ async function runTestCase(options: any, testData: any, t: tape.Test) {
   const begin = Date.now()
   // Copy the common object to not create long-lasting
   // references in memory which might prevent GC
-  const common = options.common.copy()
+  let common = options.common.copy()
   // Have to create a blockchain with empty block as genesisBlock for Merge
   // Otherwise mainnet genesis will throw since this has difficulty nonzero
   const genesisBlock = new Block(undefined, undefined, undefined, undefined, { common })
-  const blockchain = await createBlockchain({ genesisBlock, common })
-  const state = new MerklePatriciaTrie({ useKeyHashing: true, common })
-  const stateManager = new MerkleStateManager({
-    caches: new Caches(),
-    trie: state,
-    common,
-  })
+  let blockchain = await createBlockchain({ genesisBlock, common })
+  let stateTree: VerkleTree | MerklePatriciaTrie
+  let stateManager: StateManagerInterface
+  if (options.stateManager === 'verkle') {
+    const verkleCrypto = await loadVerkleCrypto()
+    stateTree = await createVerkleTree({ verkleCrypto, db: new MapDB() })
+    stateManager = new StatefulVerkleStateManager({
+      verkleCrypto,
+      trie: stateTree,
+    })
+  } else {
+    stateTree = new MerklePatriciaTrie({ useKeyHashing: true, common })
+    stateManager = new MerkleStateManager({
+      caches: new Caches(),
+      trie: stateTree,
+      common,
+    })
+  }
 
   const evmOpts = {
     bls: options.bls,
     bn254: options.bn254,
   }
-  const vm = await createVM({
+  let vm = await createVM({
     stateManager,
     common,
     blockchain,
@@ -177,9 +193,7 @@ async function runTestCase(options: any, testData: any, t: tape.Test) {
   vm.evm.events!.removeListener('step', stepHandler)
   vm.events.removeListener('afterTx', afterTxHandler)
 
-  // @ts-ignore Explicitly delete objects for memory optimization (early GC)
-  // TODO FIXME
-  //common = blockchain = state = stateManager = evm = vm = null // eslint-disable-line
+  common = blockchain = stateTree = stateManager = vm = null as any
 
   return parseFloat(timeSpent)
 }