Support for asynchronous functions, and skipping individual runs

README.md

@@ -35,6 +35,27 @@ benchmarks.add(
     { setup }
 );
 
+benchmarks.add(
+    'asynchronous',
+    async (client, { histogram }) => {
+        const result = await lookup();
+        return result.name;
+    },
+    { setup }
+);
+
+
+benchmarks.add(
+    'skippable',
+    (client, { histogram }) => histogram.observe(1, { a: 1, b: 1 }),
+    { 
+        setup,
+        start: (event) => (event.target.name !== 'prom-client@10.1.3') // function not supported or broken in this version
+    }
+);
+
+
+
 benchmarks.run().catch(err => {
     console.error(err.stack);
     process.exit(1);

src/index.js

@@ -27,18 +27,28 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) {
         for (const bucketName of Object.keys(buckets)) {
             const bucket = buckets[bucketName];
 
-
             for (const benchmarkName of Object.keys(bucket)) {
-                const { fn, opts: { setup, teardown } } = bucket[benchmarkName];
+                const { fn, opts: { setup, teardown, start = undefined } } = bucket[benchmarkName];
 
                 const fastest = { time: -Infinity };
                 const slowest = { time: +Infinity };
 
                 for (const testModule of testModules) {
                     const name = `${bucketName} ➭ ${benchmarkName} ➭ ${testModule.name}`;
-                    const ctx = await Promise.resolve(setup(testModule.module));
-                    const result = await new Promise((resolve, reject) => {
-                        const bench = new benchmark.Benchmark(name, () => fn(testModule.module, ctx));
+                    let ctx;
+
+                    const result = await new Promise(async (resolve, reject) => {
+                        const bench = new benchmark.Benchmark({
+                            name,
+                            defer: true,
+                            fn: async (done) => {
+                                try {
+                                    done.resolve(await fn(testModule.module, ctx));
+                                } catch (err) {
+                                    done.resolve(err);
+                                }
+                            }
+                        });
 
                         bench.on('complete', (event) => {
                             if (event.target.error) {
@@ -76,6 +86,20 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) {
                             }
                         });
 
+
+                        if (start !== undefined) {
+                            // fake a start() call early
+                            bench.on('start', start);
+                            const cancelled = (bench.emit('start') === false);
+                            bench.off('start', start);
+
+                            if (cancelled) {
+                                bench.emit('complete');
+                                return;
+                            }
+                        }
+
+                        ctx = await Promise.resolve(setup(testModule.module));
                         bench.run();
                     });
 

test/index.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const assert = require('assert');
 const createRegressionBenchmark = require('..');
 
 const benchmarks = createRegressionBenchmark(
@@ -29,11 +30,52 @@ benchmarks.add(
     { setup }
 );
 
-benchmarks.run().catch(err => {
-    console.error(err.stack);
-    process.exit(1);
+const selftest = createRegressionBenchmark(
+    {}, []
+);
+
+selftest.suite('async support', (suite) => {
+    suite.add(
+        'async setup',
+        (client, { completed }) => assert(completed),
+        { setup: asyncSetup },
+    );
+
+    suite.add(
+        'async fn',
+        (client, ctx) => {
+            assert(ctx.running !== true);
+            ctx.running = true;
+
+            return new Promise(resolve => setTimeout(() => {
+                ctx.running = false;
+                resolve();
+            }, 100));
+        },
+        { setup: asyncSetup },
+    );
+
+    suite.add(
+        'start callback',
+        () => {
+            assert.fail('run should have been skipped');
+        },
+        {
+            setup: () => assert.fail('Setup should have been skipped'),
+            start: (event) => {
+                return (event.target.name !== 'async support ➭ start callback ➭ current');
+            }
+        },
+    );
 });
 
+benchmarks.run()
+    .then(() => selftest.run())
+    .catch(err => {
+        console.error(err.stack);
+        process.exit(1);
+    });
+
 function setup(client) {
     const registry = new client.Registry();
 
@@ -48,3 +90,9 @@ function setup(client) {
 
     return {registry, histogram};
 }
+
+async function asyncSetup() {
+    await new Promise(resolve => setTimeout(resolve, 300));
+
+    return { completed: {} };
+}