feat: add support for post-CI automated benchmarking

packages/swingset-runner/autobench

@@ -0,0 +1,2 @@
+#!/bin/sh
+bin/runner --init --benchmark 100 --statsfile benchstats.json --config demo/exchangeBenchmark/swingset.json run -- --quiet --prime

packages/swingset-runner/demo/exchangeBenchmark/bootstrap.js

@@ -8,8 +8,20 @@ export function buildRootObject(_vatPowers, vatParameters) {
   let alice;
   let bob;
   let round = 0;
+  let quiet = false;
+
   return harden({
     async bootstrap(vats, devices) {
+      let primeContracts = false;
+      console.log(`@@ params = ${JSON.stringify(vatParameters)}`);
+      for (const arg of vatParameters.argv) {
+        if (arg === '--prime') {
+          primeContracts = true;
+        } else if (arg === '--quiet') {
+          quiet = true;
+        }
+      }
+
       const vatAdminSvc = await E(vats.vatAdmin).createVatAdminService(
         devices.vatAdmin,
       );
@@ -50,18 +62,18 @@ export function buildRootObject(_vatPowers, vatParameters) {
 
       // Zoe appears to do some one-time setup the first time it's used, so this
       // is an optional, sacrifical benchmark round to prime the pump.
-      if (vatParameters.argv[0] === '--prime') {
-        await E(alice).initiateTrade(bob);
-        await E(bob).initiateTrade(alice);
+      if (primeContracts) {
+        await E(alice).initiateTrade(bob, quiet);
+        await E(bob).initiateTrade(alice, quiet);
       }
     },
     async runBenchmarkRound() {
       round += 1;
       if (round % 2) {
-        await E(alice).initiateTrade(bob);
+        await E(alice).initiateTrade(bob, quiet);
         return `round ${round} (alice->bob) complete`;
       } else {
-        await E(bob).initiateTrade(alice);
+        await E(bob).initiateTrade(alice, quiet);
         return `round ${round} (bob->alice) complete`;
       }
     },

packages/swingset-runner/demo/exchangeBenchmark/exchanger.js

@@ -25,14 +25,16 @@ async function build(name, zoe, issuers, payments, publicFacet) {
 
   const invitationIssuer = await E(zoe).getInvitationIssuer();
 
-  async function preReport() {
-    await showPurseBalance(moolaPurseP, `${name} moola before`, log);
-    await showPurseBalance(simoleanPurseP, `${name} simoleans before`, log);
+  async function preReport(quiet) {
+    const useLog = quiet ? () => {} : log;
+    await showPurseBalance(moolaPurseP, `${name} moola before`, useLog);
+    await showPurseBalance(simoleanPurseP, `${name} simoleans before`, useLog);
   }
 
-  async function postReport() {
-    await showPurseBalance(moolaPurseP, `${name} moola after`, log);
-    await showPurseBalance(simoleanPurseP, `${name} simoleans after`, log);
+  async function postReport(quiet) {
+    const useLog = quiet ? () => {} : log;
+    await showPurseBalance(moolaPurseP, `${name} moola after`, useLog);
+    await showPurseBalance(simoleanPurseP, `${name} simoleans after`, useLog);
   }
 
   async function receivePayout(payoutP) {
@@ -44,8 +46,8 @@ async function build(name, zoe, issuers, payments, publicFacet) {
     await E(simoleanPurseP).deposit(simoleanPayout);
   }
 
-  async function initiateTrade(otherP) {
-    await preReport();
+  async function initiateTrade(otherP, quiet) {
+    await preReport(quiet);
 
     const buyOrderInvitation = await E(publicFacet).makeInvitation();
 
@@ -65,14 +67,14 @@ async function build(name, zoe, issuers, payments, publicFacet) {
     const payoutP = E(seat).getPayouts();
 
     const invitationP = E(publicFacet).makeInvitation();
-    await E(otherP).respondToTrade(invitationP);
+    await E(otherP).respondToTrade(invitationP, quiet);
 
     await receivePayout(payoutP);
-    await postReport();
+    await postReport(quiet);
   }
 
-  async function respondToTrade(invitationP) {
-    await preReport();
+  async function respondToTrade(invitationP, quiet) {
+    await preReport(quiet);
 
     const invitation = await invitationP;
     const exclInvitation = await E(invitationIssuer).claim(invitation);
@@ -94,7 +96,7 @@ async function build(name, zoe, issuers, payments, publicFacet) {
     const payoutP = E(seatP).getPayouts();
 
     await receivePayout(payoutP);
-    await postReport();
+    await postReport(quiet);
   }
 
   return harden({

packages/swingset-runner/src/kerneldump.js

@@ -7,7 +7,7 @@ import { openSwingStore as openSimpleSwingStore } from '@agoric/swing-store-simp
 
 import { dumpStore } from './dumpstore';
 import { auditRefCounts } from './auditstore';
-import { printStats } from './printStats';
+import { organizeMainStats, printMainStats } from './printStats';
 
 function usage() {
   console.log(`
@@ -130,9 +130,9 @@ export function main() {
       fail(`invalid database mode ${dbMode}`, true);
   }
   if (justStats) {
-    const stats = JSON.parse(store.storage.get('kernelStats'));
+    const rawStats = JSON.parse(store.storage.get('kernelStats'));
     const cranks = Number(store.storage.get('crankNumber'));
-    printStats(stats, cranks);
+    printMainStats(organizeMainStats(rawStats, cranks));
   } else {
     if (doDump) {
       dumpStore(store.storage, outfile, rawMode);

packages/swingset-runner/src/main.js

@@ -22,7 +22,13 @@ import {
 
 import { dumpStore } from './dumpstore';
 import { auditRefCounts } from './auditstore';
-import { printStats, printBenchmarkStats } from './printStats';
+import {
+  organizeBenchmarkStats,
+  printBenchmarkStats,
+  organizeMainStats,
+  printMainStats,
+  outputStats,
+} from './printStats';
 
 const log = console.log;
 
@@ -63,7 +69,8 @@ FLAGS may be:
   --dumpdir DIR    - place kernel state dumps in directory DIR (default ".")
   --dumptag STR    - prefix kernel state dump filenames with STR (default "t")
   --raw            - perform kernel state dumps in raw mode
-  --stats          - print performance stats at the end of a run
+  --stats          - print a performance stats report at the end of a run
+  --statsfile FILE - output performance stats to FILE as a JSON object
   --benchmark N    - perform an N round benchmark after the initial run
   --indirect       - launch swingset from a vat instead of launching directly
   --globalmetering - install metering on global objects
@@ -172,6 +179,7 @@ export async function main() {
   let launchIndirectly = false;
   let benchmarkRounds = 0;
   let configPath = null;
+  let statsFile = null;
   let dbDir = null;
   let initOnly = false;
 
@@ -248,6 +256,9 @@ export async function main() {
       case '--stats':
         shouldPrintStats = true;
         break;
+      case '--statsfile':
+        statsFile = argv.shift();
+        break;
       case '--globalmetering':
         globalMeteringActive = true;
         break;
@@ -410,6 +421,9 @@ export async function main() {
     statLogger = makeStatLogger(logTag, headers);
   }
 
+  let mainStats;
+  let benchmarkStats;
+
   let crankNumber = 0;
   switch (command) {
     case 'run': {
@@ -509,7 +523,7 @@ export async function main() {
 
   async function runBenchmark(rounds) {
     const cranksPre = getCrankNumber();
-    const statsPre = controller.getStats();
+    const rawStatsPre = controller.getStats();
     const args = { body: '[]', slots: [] };
     let totalSteps = 0;
     let totalDeltaT = 0n;
@@ -534,8 +548,14 @@ export async function main() {
       totalDeltaT += deltaT;
     }
     const cranksPost = getCrankNumber();
-    const statsPost = controller.getStats();
-    printBenchmarkStats(statsPre, statsPost, cranksPost - cranksPre, rounds);
+    const rawStatsPost = controller.getStats();
+    benchmarkStats = organizeBenchmarkStats(
+      rawStatsPre,
+      rawStatsPost,
+      cranksPost - cranksPre,
+      rounds,
+    );
+    printBenchmarkStats(benchmarkStats);
     return [totalSteps, totalDeltaT];
   }
 
@@ -626,9 +646,11 @@ export async function main() {
     if (!runInBlockMode) {
       store.commit();
     }
+    const cranks = getCrankNumber();
+    const rawStats = controller.getStats();
+    mainStats = organizeMainStats(rawStats, cranks);
     if (shouldPrintStats) {
-      const cranks = getCrankNumber();
-      printStats(controller.getStats(), cranks);
+      printMainStats(mainStats);
     }
     if (benchmarkRounds > 0) {
       const [moreSteps, moreDeltaT] = await runBenchmark(benchmarkRounds);
@@ -651,6 +673,9 @@ export async function main() {
       }
     }
     store.close();
+    if (statsFile) {
+      outputStats(statsFile, mainStats, benchmarkStats);
+    }
     if (totalSteps) {
       const per = deltaT / BigInt(totalSteps);
       log(

packages/swingset-runner/src/printStats.js

@@ -1,3 +1,5 @@
+import fs from 'fs';
+
 const log = console.log;
 
 // Return a string representation of a number with 4 digits of precision to the
@@ -16,7 +18,29 @@ function isMainKey(key) {
   return !key.endsWith('Max') && !key.endsWith('Up') && !key.endsWith('Down');
 }
 
-export function printStats(stats, cranks) {
+export function organizeMainStats(rawStats, cranks) {
+  const stats = {
+    cranks,
+    data: {},
+  };
+  for (const [key, value] of Object.entries(rawStats)) {
+    if (isMainKey(key)) {
+      const upKey = `${key}Up`;
+      const downKey = `${key}Down`;
+      const maxKey = `${key}Max`;
+      stats.data[key] = {
+        value,
+        up: rawStats[upKey],
+        down: rawStats[downKey],
+        max: rawStats[maxKey],
+        perCrank: value / cranks,
+      };
+    }
+  }
+  return stats;
+}
+
+export function printMainStats(stats) {
   const w1 = 32;
   const h1 = `${'Stat'.padEnd(w1)}`;
   const d1 = `${''.padEnd(w1, '-')}`;
@@ -41,36 +65,53 @@ export function printStats(stats, cranks) {
   const h6 = ` ${'PerCrank'.padStart(w6)}`;
   const d6 = ` ${''.padStart(w6, '-')}`;
 
-  log(`In ${cranks} cranks:`);
+  log(`In ${stats.cranks} cranks:`);
   log(`${h1} ${h2} ${h3} ${h4} ${h5} ${h6}`);
   log(`${d1} ${d2} ${d3} ${d4} ${d5} ${d6}`);
 
-  for (const [key, value] of Object.entries(stats)) {
-    if (isMainKey(key)) {
-      const col1 = `${key.padEnd(w1)}`;
+  const data = stats.data;
+  for (const [key, entry] of Object.entries(data)) {
+    const col1 = `${key.padEnd(w1)}`;
 
-      const col2 = `${String(value).padStart(w2)}`;
+    const col2 = `${String(entry.value).padStart(w2)}`;
 
-      const upKey = `${key}Up`;
-      const v3 = stats[upKey] !== undefined ? stats[upKey] : '';
-      const col3 = `${String(v3).padStart(w3)}`;
+    const v3 = entry.up !== undefined ? entry.up : '';
+    const col3 = `${String(v3).padStart(w3)}`;
 
-      const downKey = `${key}Down`;
-      const v4 = stats[downKey] !== undefined ? stats[downKey] : '';
-      const col4 = `${String(v4).padStart(w4)}`;
+    const v4 = entry.down !== undefined ? entry.down : '';
+    const col4 = `${String(v4).padStart(w4)}`;
 
-      const maxKey = `${key}Max`;
-      const v5 = stats[maxKey] !== undefined ? stats[maxKey] : '';
-      const col5 = `${String(v5).padStart(w5)}`;
+    const v5 = entry.max !== undefined ? entry.max : '';
+    const col5 = `${String(v5).padStart(w5)}`;
+
+    const col6 = `${pn(entry.value / stats.cranks).padStart(w6)}`;
+
+    log(`${col1} ${col2} ${col3} ${col4} ${col5}  ${col6}`);
+  }
+}
 
-      const col6 = `${pn(value / cranks).padStart(w6)}`;
+export function organizeBenchmarkStats(rawBefore, rawAfter, cranks, rounds) {
+  const stats = {
+    cranks,
+    rounds,
+    cranksPerRound: cranks / rounds,
+    data: {},
+  };
 
-      log(`${col1} ${col2} ${col3} ${col4} ${col5}  ${col6}`);
+  // Note: the following assumes rawBefore and rawAfter have the same keys.
+  for (const [key, value] of Object.entries(rawBefore)) {
+    if (isMainKey(key)) {
+      const delta = rawAfter[key] - value;
+      stats.data[key] = {
+        delta,
+        deltaPerRound: delta / rounds,
+      };
     }
   }
+  return stats;
 }
 
-export function printBenchmarkStats(statsBefore, statsAfter, cranks, rounds) {
+export function printBenchmarkStats(stats) {
   const w1 = 32;
   const h1 = `${'Stat'.padEnd(w1)}`;
   const d1 = `${''.padEnd(w1, '-')}`;
@@ -84,18 +125,20 @@ export function printBenchmarkStats(statsBefore, statsAfter, cranks, rounds) {
   const d3 = ` ${''.padStart(w3 - 1, '-')}`;
 
   // eslint-disable-next-line prettier/prettier
-  log(`In ${cranks} cranks over ${rounds} rounds (${pn(cranks/rounds).trim()} cranks/round):`);
+  log(`In ${stats.cranks} cranks over ${stats.rounds} rounds (${pn(stats.cranksPerRound).trim()} cranks/round):`);
   log(`${h1} ${h2} ${h3}`);
   log(`${d1} ${d2} ${d3}`);
 
-  // Note: the following assumes statsBefore and statsAfter have the same keys.
-  for (const [key, value] of Object.entries(statsBefore)) {
-    if (isMainKey(key)) {
-      const col1 = `${key.padEnd(w1)}`;
-      const delta = statsAfter[key] - value;
-      const col2 = `${String(delta).padStart(w2)}`;
-      const col3 = `${pn(delta / rounds).padStart(w3)}`;
-      log(`${col1} ${col2} ${col3}`);
-    }
+  const data = stats.data;
+  for (const [key, entry] of Object.entries(data)) {
+    const col1 = `${key.padEnd(w1)}`;
+    const col2 = `${String(entry.delta).padStart(w2)}`;
+    const col3 = `${pn(entry.deltaPerRound).padStart(w3)}`;
+    log(`${col1} ${col2} ${col3}`);
   }
 }
+
+export function outputStats(statsFile, main, benchmark) {
+  const str = JSON.stringify({ main, benchmark }, undefined, 2);
+  fs.writeFileSync(statsFile, str);
+}