Add benchmark for bytecode execution

packages/vm/DEVELOPER.md

@@ -208,6 +208,12 @@ Then:
 
 To define the number of samples to be run pass in a number like so: `npm run benchmarks -- mainnetBlocks:10`
 
+To benchmark individual bytecode against empty state:
+`node --max-old-space-size=4096 ./benchmarks/run.js benchmarks bytecode:10 -b 60FF60005261FFFF600020`
+
+To benchmark individual bytecode against a state:
+`node --max-old-space-size=4096 ./benchmarks/run.js benchmarks bytecode:10 -b 60FF60005261FFFF600020 -p prestate.json`
+
 If you want to get a more detailed look to find bottlenecks we can use [0x](https://github.com/davidmarkclements/0x):
 
 ```

packages/vm/benchmarks/bytecode.ts

@@ -0,0 +1,93 @@
+import { readFileSync } from 'fs'
+import Benchmark from 'benchmark'
+import { Chain, Common, Hardfork } from '@ethereumjs/common'
+import { getPreState } from './util'
+import { EVM } from '@ethereumjs/evm'
+import { Address, hexToBytes } from '@ethereumjs/util'
+import { DefaultStateManager } from '@ethereumjs/statemanager'
+
+class Stats {
+  runId?: number
+  iterationsCount!: number
+  totalTimeNs!: number
+  stdDevTimeNs!: number
+}
+
+type EvmCreator = () => Promise<EVM>
+
+async function runBenchmark(bytecode: string, createEvm: EvmCreator): Promise<Stats> {
+  const evm = await createEvm()
+
+  let promiseResolve: any
+  const resultPromise: Promise<Stats> = new Promise((resolve, reject) => {
+    promiseResolve = resolve
+  })
+
+  const bytecodeHex = hexToBytes('0x' + bytecode)
+  const gasLimit = BigInt(0xffff)
+
+  const bench = new Benchmark({
+    defer: true,
+    maxTime: 0.5,
+    name: `Running Opcodes`,
+    fn: async (deferred: any) => {
+      try {
+        await evm.runCode({
+          code: bytecodeHex,
+          gasLimit: gasLimit,
+        })
+        deferred.resolve()
+      } catch (err) {
+        console.log('ERROR', err)
+      }
+    },
+    onCycle: (event: any) => {
+      evm.stateManager.clearContractStorage(Address.zero())
+    },
+  })
+    .on('complete', () => {
+      promiseResolve({
+        iterationsCount: bench.count,
+        totalTimeNs: Math.round(bench.stats.mean * 1_000_000_000),
+        stdDevTimeNs: Math.round(bench.stats.deviation * 1_000_000_000),
+      })
+    })
+    .run()
+
+  return resultPromise
+}
+
+export async function bytecode(
+  suite?: Benchmark.Suite,
+  numSamples?: number,
+  bytecode?: string,
+  preState?: string
+) {
+  const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Cancun })
+  let stateManager = new DefaultStateManager()
+
+  if (preState) {
+    let preStateData = JSON.parse(readFileSync(preState, 'utf8'))
+    stateManager = await getPreState(preStateData, common)
+  }
+
+  const createEvm: EvmCreator = async () => {
+    const profiler = { enabled: suite ? false : true }
+    return await EVM.create({ stateManager, common, profiler })
+  }
+
+  for (let i = 0; i < (numSamples ?? 1); i++) {
+    if (suite) {
+      let results = await runBenchmark(bytecode!, createEvm)
+      console.log(
+        `${i}, ${results.iterationsCount}, ${results.totalTimeNs}, ${results.stdDevTimeNs}`
+      )
+    } else {
+      let evm = await createEvm()
+      evm.runCode({
+        code: hexToBytes('0x' + bytecode!),
+        gasLimit: BigInt(0xffff),
+      })
+    }
+  }
+}

packages/vm/benchmarks/run.ts

@@ -1,52 +1,58 @@
 import Benchmark from 'benchmark'
 import { BenchmarksType } from './util'
 import { mainnetBlocks } from './mainnetBlocks'
+import { bytecode } from './bytecode'
+import { ArgumentParser } from 'argparse'
 
 // Add an import and a BENCHMARKS entry to list a new benchmark
 const BENCHMARKS: BenchmarksType = {
   mainnetBlocks: {
     function: mainnetBlocks,
   },
+  bytecode: {
+    function: bytecode,
+  },
 }
 
 const onCycle = (event: Benchmark.Event) => {
   console.log(String(event.target))
 }
 
 async function main() {
-  const args = process.argv
-
-  // Input validation
-  if (args.length < 4) {
-    console.log(
-      'Please provide at least one benchmark name when running a benchmark or doing profiling.'
-    )
-    console.log(
-      'Usage: npm run benchmarks|profiling -- BENCHMARK_NAME[:NUM_SAMPLES][,BENCHMARK_NAME[:NUM_SAMPLES]]'
-    )
-    console.log(`Benchmarks available: ${Object.keys(BENCHMARKS).join(', ')}`)
-    return process.exit(1)
-  }
+  const parser = new ArgumentParser({ description: 'Benchmark arbitrary bytecode.' })
+  parser.add_argument('mode', {
+    help: 'Mode of running',
+    choices: ['benchmarks', 'profiling'],
+    type: 'str',
+  })
+  parser.add_argument('benchmarks', {
+    help: `Name(s) of benchmaks to run: BENCHMARK_NAME[:NUM_SAMPLES][,BENCHMARK_NAME[:NUM_SAMPLES]]. Benchmarks available: ${Object.keys(
+      BENCHMARKS
+    ).join(', ')}`,
+    type: 'str',
+  })
+  parser.add_argument('-b', '--bytecode', { help: 'Bytecode to run', type: 'str' })
+  parser.add_argument('-p', '--preState', { help: 'File containing prestate to load', type: 'str' })
+  const args = parser.parse_args()
 
   // Initialization
   let suite
   // Choose between benchmarking or profiling (with 0x)
-  if (args[2] === 'benchmarks') {
+  if (args.mode === 'benchmarks') {
     console.log('Benchmarking started...')
     suite = new Benchmark.Suite()
   } else {
     console.log('Profiling started...')
   }
 
-  const benchmarks = args[3].split(',')
-
   // Benchmark execution
+  const benchmarks = args.benchmarks.split(',')
   for (const benchmark of benchmarks) {
     const [name, numSamples] = benchmark.split(':')
 
     if (name in BENCHMARKS) {
       console.log(`Running '${name}':`)
-      await BENCHMARKS[name].function(suite, Number(numSamples))
+      await BENCHMARKS[name].function(suite, Number(numSamples), args.bytecode, args.preState)
     } else {
       console.log(`Benchmark with name ${name} doesn't exist, skipping...`)
     }