Add save/compare functionality to the benchmarker

Sources/RegexBenchmark/Benchmark.swift

@@ -51,76 +51,6 @@ public struct NSBenchmark: RegexBenchmark {
   }
 }
 
-public struct BenchmarkRunner {
-  // Register instances of Benchmark and run them
-  let suiteName: String
-  var suite: [any RegexBenchmark]
-  let samples: Int
-
-  public init(_ suiteName: String) {
-    self.init(suiteName, 20)
-  }
-
-  public init(_ suiteName: String, _ n: Int) {
-    self.suiteName = suiteName
-    self.suite = []
-    self.samples = n
-  }
-
-  public mutating func register(_ new: some RegexBenchmark) {
-    suite.append(new)
-  }
-
-  func measure(benchmark: some RegexBenchmark) -> Time {
-    var times: [Time] = []
-
-    // initial run to make sure the regex has been compiled
-    // todo: measure compile times, or at least how much this first run
-    //       differs from the later ones
-    benchmark.run()
-
-    // fixme: use suspendingclock?
-    for _ in 0..<samples {
-      let start = Tick.now
-      benchmark.run()
-      let end = Tick.now
-      let time = end.elapsedTime(since: start)
-      times.append(time)
-    }
-    // todo: compute stdev and warn if it's too large
-
-    // return median time
-    times.sort()
-    return times[samples/2]
-  }
-
-  public func run() {
-    print("Running")
-    for b in suite {
-      print("- \(b.name) \(measure(benchmark: b))")
-    }
-  }
-
-  public func profile() {
-    print("Starting")
-    for b in suite {
-      print("- \(b.name)")
-      b.run()
-      print("- done")
-    }
-  }
-
-  public func debug() {
-    print("Debugging")
-    print("========================")
-    for b in suite {
-      print("- \(b.name) \(measure(benchmark: b))")
-      b.debug()
-      print("========================")
-    }
-  }
-}
-
 /// A benchmark meant to be ran across multiple engines
 struct CrossBenchmark {
   /// The base name of the benchmark

Sources/RegexBenchmark/BenchmarkRunner.swift

@@ -0,0 +1,165 @@
+import Foundation
+
+public struct BenchmarkRunner {
+  let suiteName: String
+  var suite: [any RegexBenchmark] = []
+  let samples: Int
+  var results: SuiteResult = SuiteResult()
+
+  // Outputting
+  let startTime = Date()
+  let outputPath: String
+
+  public init(_ suiteName: String, _ n: Int, _ outputPath: String) {
+    self.suiteName = suiteName
+    self.samples = n
+    self.outputPath = outputPath
+  }
+
+  public mutating func register(_ new: some RegexBenchmark) {
+    suite.append(new)
+  }
+
+  mutating func measure(benchmark: some RegexBenchmark) -> Time {
+    var times: [Time] = []
+
+    // initial run to make sure the regex has been compiled
+    // todo: measure compile times, or at least how much this first run
+    //       differs from the later ones
+    benchmark.run()
+
+    // fixme: use suspendingclock?
+    for _ in 0..<samples {
+      let start = Tick.now
+      benchmark.run()
+      let end = Tick.now
+      let time = end.elapsedTime(since: start)
+      times.append(time)
+    }
+    // todo: compute stdev and warn if it's too large
+
+    // return median time
+    times.sort()
+    let median =  times[samples/2]
+    self.results.add(name: benchmark.name, time: median)
+    return median
+  }
+
+  public mutating func run() {
+    print("Running")
+    for b in suite {
+      print("- \(b.name) \(measure(benchmark: b))")
+    }
+  }
+
+  public func profile() {
+    print("Starting")
+    for b in suite {
+      print("- \(b.name)")
+      b.run()
+      print("- done")
+    }
+  }
+
+  public mutating func debug() {
+    print("Debugging")
+    print("========================")
+    for b in suite {
+      print("- \(b.name) \(measure(benchmark: b))")
+      b.debug()
+      print("========================")
+    }
+  }
+}
+
+extension BenchmarkRunner {
+  var dateStyle: Date.FormatStyle {
+    Date.FormatStyle()
+      .year(.twoDigits)
+      .month(.twoDigits)
+      .day(.twoDigits)
+      .hour(.twoDigits(amPM: .omitted))
+      .minute(.twoDigits)
+  }
+
+  var outputFolderUrl: URL {
+    let url = URL(fileURLWithPath: outputPath, isDirectory: true)
+    if !FileManager.default.fileExists(atPath: url.path) {
+      try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true)
+    }
+    return url
+  }
+
+  public func save() throws {
+    let now = startTime.formatted(dateStyle)
+    let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json")
+    print("Saving result to \(resultJsonUrl.path)")
+    try results.save(to: resultJsonUrl)
+  }
+
+  public func compare() throws {
+    var pastResults: [Date: SuiteResult] = [:]
+    for resultFile in try FileManager.default.contentsOfDirectory(
+      at: outputFolderUrl,
+      includingPropertiesForKeys: nil
+    ) {
+      let dateString = resultFile.lastPathComponent.replacingOccurrences(
+        of: "-result.json",
+        with: "")
+      let date = try dateStyle.parse(dateString)
+      pastResults.updateValue(try SuiteResult.load(from: resultFile), forKey: date)
+    }
+    let sorted = pastResults
+      .filter({kv in startTime.timeIntervalSince(kv.0) > 1})
+      .sorted(by: {(kv1,kv2) in kv1.0 > kv2.0})
+    let latest = sorted[0]
+    let diff = results.compare(with: latest.1)
+    let regressions = diff.filter({kv in kv.1.seconds > 0})
+    let improvements = diff.filter({kv in kv.1.seconds < 0})
+
+    print("Comparing against benchmark done on \(latest.0.formatted(dateStyle))")
+    print("=== Regressions ===")
+    for item in regressions {
+      print("- \(item.key) \(item.value)")
+    }
+    print("=== Improvements ===")
+    for item in improvements {
+      print("- \(item.key) \(item.value)")
+    }
+  }
+}
+
+struct SuiteResult {
+  var results: [String: Time] = [:]
+
+  public mutating func add(name: String, time: Time) {
+    results.updateValue(time, forKey: name)
+  }
+
+  public func compare(with other: SuiteResult) -> [String: Time] {
+    var output: [String: Time] = [:]
+    for item in results {
+      if let otherVal = other.results[item.key] {
+        let diff = item.value - otherVal
+        if diff.abs() > Time.millisecond{
+          output.updateValue(diff, forKey: item.key)
+        }
+      }
+    }
+    return output
+  }
+}
+
+extension SuiteResult: Codable {
+  public func save(to url: URL) throws {
+    let encoder = JSONEncoder()
+    let data = try encoder.encode(self)
+    try data.write(to: url, options: .atomic)
+  }
+
+  public static func load(from url: URL) throws -> SuiteResult {
+    let decoder = JSONDecoder()
+    let data = try Data(contentsOf: url)
+    return try decoder.decode(SuiteResult.self, from: data)
+  }
+}

Sources/RegexBenchmark/CLI.swift

@@ -13,9 +13,18 @@ struct Runner: ParsableCommand {
 
   @Option(help: "Debug benchmark regexes")
   var debug = false
+
+  @Option(help: "Output folder")
+  var outputPath = "./results/"
+
+  @Option(help: "Should the results be saved")
+  var save = false
+
+  @Option(help: "Compare this result with the latest saved result")
+  var compare = false
 
   func makeRunner() -> BenchmarkRunner {
-    var benchmark = BenchmarkRunner("RegexBench", samples)
+    var benchmark = BenchmarkRunner("RegexBench", samples, outputPath)
     benchmark.addReluctantQuant()
     benchmark.addCSS()
     benchmark.addNotFound()
@@ -35,7 +44,14 @@ struct Runner: ParsableCommand {
     case (true, true): print("Cannot run both profile and debug")
     case (true, false): runner.profile()
     case (false, true): runner.debug()
-    case (false, false): runner.run()
+    case (false, false):
+      runner.run()
+      if save {
+        try runner.save()
+      }
+      if compare {
+        try runner.compare()
+      }
     }
   }
 }

Sources/RegexBenchmark/Suite/CssRegex.swift

@@ -4,9 +4,6 @@ import _StringProcessing
 extension BenchmarkRunner {
   mutating func addCSS() {
     let r = #"--([a-zA-Z0-9_-]+)\s*:\s*(.*?);"#
-
-    // FIXME: Why is `first` and `all` the same running time?
-
     let css = CrossBenchmark(
       baseName: "css", regex: r, input: Inputs.swiftOrgCSS)
     css.register(&self)

Sources/RegexBenchmark/Utils/Time.swift

@@ -55,28 +55,38 @@ extension Time: Comparable {
   }
 }
 
+extension Time {
+  public static func - (left: Self, right: Self) -> Self {
+    return Time(left.seconds - right.seconds)
+  }
+
+  public func abs() -> Time {
+    Time(Swift.abs(self.seconds))
+  }
+}
+
 extension Time: CustomStringConvertible {
   public var description: String {
     if self.seconds == 0 { return "0" }
-    if self < .attosecond { return String(format: "%.3gas", seconds * 1e18) }
-    if self < .picosecond { return String(format: "%.3gfs", seconds * 1e15) }
-    if self < .nanosecond  { return String(format: "%.3gps", seconds * 1e12) }
-    if self < .microsecond { return String(format: "%.3gns", seconds * 1e9) }
-    if self < .millisecond { return String(format: "%.3gµs", seconds * 1e6) }
-    if self < .second      { return String(format: "%.3gms", seconds * 1e3) }
+    if self.abs() < .attosecond { return String(format: "%.3gas", seconds * 1e18) }
+    if self.abs() < .picosecond { return String(format: "%.3gfs", seconds * 1e15) }
+    if self.abs() < .nanosecond  { return String(format: "%.3gps", seconds * 1e12) }
+    if self.abs() < .microsecond { return String(format: "%.3gns", seconds * 1e9) }
+    if self.abs() < .millisecond { return String(format: "%.3gµs", seconds * 1e6) }
+    if self.abs() < .second      { return String(format: "%.3gms", seconds * 1e3) }
     if self.seconds < 1000 { return String(format: "%.3gs", seconds) }
     return String(format: "%gs", seconds.rounded())
   }
 
   public var typesetDescription: String {
     let spc = "\u{200A}"
     if self.seconds == 0 { return "0\(spc)s" }
-    if self < .femtosecond { return String(format: "%.3g\(spc)as", seconds * 1e18) }
-    if self < .picosecond { return String(format: "%.3g\(spc)fs", seconds * 1e15) }
-    if self < .nanosecond  { return String(format: "%.3g\(spc)ps", seconds * 1e12) }
-    if self < .microsecond { return String(format: "%.3g\(spc)ns", seconds * 1e9) }
-    if self < .millisecond { return String(format: "%.3g\(spc)µs", seconds * 1e6) }
-    if self < .second      { return String(format: "%.3g\(spc)ms", seconds * 1e3) }
+    if self.abs() < .femtosecond { return String(format: "%.3g\(spc)as", seconds * 1e18) }
+    if self.abs() < .picosecond { return String(format: "%.3g\(spc)fs", seconds * 1e15) }
+    if self.abs() < .nanosecond  { return String(format: "%.3g\(spc)ps", seconds * 1e12) }
+    if self.abs() < .microsecond { return String(format: "%.3g\(spc)ns", seconds * 1e9) }
+    if self.abs() < .millisecond { return String(format: "%.3g\(spc)µs", seconds * 1e6) }
+    if self.abs() < .second      { return String(format: "%.3g\(spc)ms", seconds * 1e3) }
     if self.seconds < 1000 { return String(format: "%.3g\(spc)s", seconds) }
     return String(format: "%g\(spc)s", seconds.rounded())
   }