Merge pull request #43 from jberryman/dev

Add new flag, hdr-histogram fixes, add 'min'

Author: jberryman

app/cli/src/commands/query.ts

@@ -37,6 +37,11 @@ export default class Query extends Command {
       multiple: false,
       description: 'URL to direct graphql queries; may override \'url\' from the YAML config, which is optional if this flag is passed',
     }),
+    query: flags.string({
+      required: false,
+      multiple: false,
+      description: 'A specific named query to run from the config; if omitted, all queries will be run',
+    }),
   }
 
   async run() {
@@ -49,7 +54,7 @@ export default class Query extends Command {
       config.url = flags.url
     }
     const executor = new BenchmarkRunner(config)
-    const results = await executor.runBenchmarks()
+    const results = await executor.runBenchmarks(flags.query)
 
     if (flags.outfile) {
       const pathToOutfile = path.join(process.cwd(), flags.outfile)

app/queries/src/PreciseHdrHistogram.ts

@@ -0,0 +1,85 @@
+import * as hdr from 'hdr-histogram-js'
+import { parseHdrHistogramText } from './executors/base/index'
+import { HDRHistogramParsedStats } from './executors/base/types'
+
+/* A wrapper for Histogram that gets us more precision. See
+ * https://github.com/HdrHistogram/HdrHistogramJS/issues/35
+ *
+ * Values inserted will be truncated to `logBase 10 scalingFactor` (i.e. 3)
+ * decimal places.
+ */
+export class PreciseHdrHistogram {
+  // We'll need to multiply by scalingFactor anything we insert, and divide by scalingFactor
+  // anything we output from here:
+  private _histogramDirty: hdr.Histogram
+  static scalingFactor: number = 1000
+
+  constructor(
+    request: hdr.BuildRequest
+  ) {
+    this._histogramDirty = hdr.build(request)
+  }
+
+  //// On inputs we multiply...
+  public recordValue(value: number) {
+    this._histogramDirty.recordValue(value*PreciseHdrHistogram.scalingFactor)
+  }
+
+  public recordValueWithCount(value: number, count: number): void {
+    this._histogramDirty.recordValueWithCount(value*PreciseHdrHistogram.scalingFactor, count)
+  }
+
+  //// ...and on outputs we divide:
+  public toJSON(): hdr.HistogramSummary {
+    let summary = this._histogramDirty.summary
+    for (let key in summary) {
+      // scale mean and percentiles (but not counts) back down:
+      if (key == "totalCount") continue
+      summary[key] /= PreciseHdrHistogram.scalingFactor
+    }
+
+    return summary
+  }
+  get mean(): number {
+    return (this._histogramDirty.mean / PreciseHdrHistogram.scalingFactor)
+  }
+  get min(): number {
+    // NOTE: 'minNonZeroValue' is already just 'min' since 0 can't be recorded
+    return (this._histogramDirty.minNonZeroValue / PreciseHdrHistogram.scalingFactor)
+  }
+  get stdDeviation(): number {
+    return (this._histogramDirty.stdDeviation / PreciseHdrHistogram.scalingFactor)
+  }
+  // This is our own helper, where formerly we called:
+  //    parseHdrHistogramText(histogram.outputPercentileDistribution())
+  get parsedStats(): HDRHistogramParsedStats[] {
+    let parsedDirty = parseHdrHistogramText(
+      this._histogramDirty.outputPercentileDistribution())
+
+    // scale mean and percentiles (but not counts) back down:
+    parsedDirty.forEach(function (line) { 
+      // i.e. line.value /= PreciseHdrHistogram.scalingFactor
+      line.value = String((Number(line.value) / PreciseHdrHistogram.scalingFactor))
+    })
+    return parsedDirty
+  }
+
+  // Don't leak implementation in debugging output, which might be confusing:
+  [Symbol.for("nodejs.util.inspect.custom")]() {
+    return JSON.stringify(this.toJSON(), null, 2)
+  }
+}
+
+// Copy-pasted from 'hdr-histogram-js', since this isn't exported
+export const defaultRequest: hdr.BuildRequest = {
+  bitBucketSize: 32,
+  autoResize: true,
+  lowestDiscernibleValue: 1,
+  highestTrackableValue: 2,
+  numberOfSignificantValueDigits: 3,
+  useWebAssembly: false,
+}
+
+export const build = (request = defaultRequest): PreciseHdrHistogram => {
+  return new PreciseHdrHistogram(request) 
+}

app/queries/src/executors/autocannon/index.ts

@@ -24,7 +24,7 @@ import { Options as AutocannonOptions } from 'autocannon'
 
 import fs from 'fs-extra'
 import path from 'path'
-import * as hdr from 'hdr-histogram-js'
+import * as hdr from '../../PreciseHdrHistogram'
 
 export class AutocannonExecutor extends BenchmarkExecutor {
   public tool = BenchmarkTool.AUTOCANNON

app/queries/src/executors/base/index.ts

@@ -21,7 +21,8 @@ export function parseHdrHistogramText(text: string): HDRHistogramParsedStats[] {
   const lines = text.split('\n')
   for (let line of lines) {
     let entries = line.trim().split(/\s+/)
-    let valid = entries.length == 4 && entries.every(Number)
+    // Careful of truthiness when parsing zero here:
+    let valid = entries.length == 4 && entries.every((x) => !(isNaN(Number(x))))
     if (!valid) continue
     let [value, percentile, totalCount, ofOnePercentile] = entries
     results.push({ value, percentile, totalCount, ofOnePercentile })
@@ -42,12 +43,10 @@ export function makeBenchmarkMetrics(
       json: {
         ...histogram.toJSON(),
         mean: histogram.mean,
+        min: histogram.min,
         stdDeviation: histogram.stdDeviation,
       },
-      text: histogram.outputPercentileDistribution(),
-      parsedStats: parseHdrHistogramText(
-        histogram.outputPercentileDistribution()
-      ),
+      parsedStats: histogram.parsedStats,
     },
   }
 }

app/queries/src/executors/base/types.ts

@@ -2,6 +2,7 @@ import type * as stream from 'stream'
 
 import type Histogram from 'hdr-histogram-js/src/Histogram'
 import type { HistogramSummary } from 'hdr-histogram-js/src/Histogram'
+import * as precise_hdr from '../../PreciseHdrHistogram'
 
 import type { K6Options } from '../k6/types'
 import type { Stage as K6Stage } from 'k6/options'
@@ -122,15 +123,17 @@ export type Benchmark =
  * ========================
  */
 
+// see 'parseHdrHistogramText()'
 export interface HDRHistogramParsedStats {
   value: string
   percentile: string
   totalCount: string
   ofOnePercentile: string
 }
 
-export interface HistogramSummaryWithMeanAndStdDev extends HistogramSummary {
+export interface HistogramSummaryWithMeanMinAndStdDev extends HistogramSummary {
   mean: number
+  min: number
   stdDeviation: number
 }
 
@@ -149,15 +152,14 @@ export interface BenchmarkMetrics {
     bytesPerSecond: number
   }
   histogram: {
-    json: HistogramSummaryWithMeanAndStdDev
-    text: string
+    json: HistogramSummaryWithMeanMinAndStdDev
     parsedStats: HDRHistogramParsedStats[]
   }
 }
 
 export interface BenchmarkMetricParams {
   name: string
-  histogram: Histogram
+  histogram: precise_hdr.PreciseHdrHistogram
   time: {
     start: Date | string
     end: Date | string

app/queries/src/executors/k6/index.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs-extra'
 import * as path from 'path'
 import * as cp from 'child_process'
-import * as hdr from 'hdr-histogram-js'
+import * as hdr from '../../PreciseHdrHistogram'
 import readline from 'readline'
 
 import {

app/queries/src/executors/wrk2/index.ts

@@ -52,12 +52,12 @@ export class Wrk2Executor extends BenchmarkExecutor {
         end: end.toISOString(),
       },
       histogram: {
-        text: hdrHistogramStdout,
         parsedStats: parseHdrHistogramText(hdrHistogramStdout),
         json: {
           totalCount: stats.requests,
           max: stats.latency_aggregate.max,
           mean: stats.latency_aggregate.mean,
+          min: stats.latency_aggregate.min,
           stdDeviation: stats.latency_aggregate.stdev,
           p50: findPercentile(stats, 50).latency_in_milliseconds,
           p75: findPercentile(stats, 75).latency_in_milliseconds,

app/queries/src/main.ts

@@ -11,10 +11,13 @@ import {
 export class BenchmarkRunner {
   constructor(public config: GlobalConfig) {}
 
-  public async runBenchmarks() {
+  public async runBenchmarks(only_query?: string) {
     let results: BenchmarkMetrics[] = []
 
     for (let query of this.config.queries) {
+      // Maybe run just a single requested benchmark from the config:
+      if (only_query && query.name != only_query) continue
+
       for (let tool of query.tools) {
         switch (tool) {
           case BenchmarkTool.AUTOCANNON: {

app/queries/src/utils.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs'
 import readline from 'readline'
 
-import * as hdr from 'hdr-histogram-js'
+import * as hdr from './PreciseHdrHistogram'
 
 async function* parseNDJSON(filepath: string) {
   const filestream = fs.createReadStream(filepath)