Allow specifying number of iterations via --benchmark_min_time.

include/benchmark/benchmark.h

@@ -1081,11 +1081,12 @@ class BENCHMARK_EXPORT Benchmark {
   Benchmark* MinWarmUpTime(double t);
 
   // Specify the amount of iterations that should be run by this benchmark.
+  // This option overrides the `benchmark_min_time` flag.
   // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark.
   //
   // NOTE: This function should only be used when *exact* iteration control is
   //   needed and never to control or limit how long a benchmark runs, where
-  // `--benchmark_min_time=N` or `MinTime(...)` should be used instead.
+  // `--benchmark_min_time=<N>s` or `MinTime(...)` should be used instead.
   Benchmark* Iterations(IterationCount n);
 
   // Specify the amount of times to repeat this benchmark. This option overrides

src/benchmark.cc

@@ -65,12 +65,21 @@ BM_DEFINE_bool(benchmark_list_tests, false);
 // linked into the binary are run.
 BM_DEFINE_string(benchmark_filter, "");
 
-// Minimum number of seconds we should run benchmark before results are
-// considered significant.  For cpu-time based tests, this is the lower bound
+// Specification of how long to run the benchmark.
+//
+// It can be either an exact number of iterations (specified as `<integer>x`),
+// or a minimum number of seconds (specified as `<float>s`). If the latter
+// format (ie., min seconds) is used, the system may run the benchmark longer
+// until the results are considered significant.
+//
+// For backward compatibility, the `s` suffix may be omitted, in which case,
+// the specified number is interpreted as the number of seconds.
+//
+// For cpu-time based tests, this is the lower bound
 // on the total cpu time used by all threads that make up the test.  For
 // real-time based tests, this is the lower bound on the elapsed time of the
 // benchmark execution, regardless of number of threads.
-BM_DEFINE_double(benchmark_min_time, 0.5);
+BM_DEFINE_string(benchmark_min_time, "0.5s");
 
 // Minimum number of seconds a benchmark should be run before results should be
 // taken into account. This e.g can be necessary for benchmarks of code which
@@ -605,7 +614,7 @@ void ParseCommandLineFlags(int* argc, char** argv) {
     if (ParseBoolFlag(argv[i], "benchmark_list_tests",
                       &FLAGS_benchmark_list_tests) ||
         ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) ||
-        ParseDoubleFlag(argv[i], "benchmark_min_time",
+        ParseStringFlag(argv[i], "benchmark_min_time",
                         &FLAGS_benchmark_min_time) ||
         ParseDoubleFlag(argv[i], "benchmark_min_warmup_time",
                         &FLAGS_benchmark_min_warmup_time) ||
@@ -666,7 +675,7 @@ void PrintDefaultHelp() {
           "benchmark"
           " [--benchmark_list_tests={true|false}]\n"
           "          [--benchmark_filter=<regex>]\n"
-          "          [--benchmark_min_time=<min_time>]\n"
+          "          [--benchmark_min_time=`<integer>x` OR `<float>s` ]\n"
           "          [--benchmark_min_warmup_time=<min_warmup_time>]\n"
           "          [--benchmark_repetitions=<num_repetitions>]\n"
           "          [--benchmark_enable_random_interleaving={true|false}]\n"

src/benchmark_runner.cc

@@ -140,23 +140,102 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters,
   manager->NotifyThreadComplete();
 }
 
+struct BenchTimeType {
+  enum { ITERS, TIME } tag;
+  union {
+    int iters;
+    double time;
+  };
+};
+
+BenchTimeType ParseBenchMinTime(const std::string& value) {
+  BenchTimeType ret;
+
+  if (value.empty()) {
+    ret.tag = BenchTimeType::TIME;
+    ret.time = 0.0;
+    return ret;
+  }
+
+  if (value.back() == 'x') {
+    std::string num_iters_str = value.substr(0, value.length() - 1);
+    int num_iters = std::atoi(num_iters_str.c_str());
+
+    // std::atoi doesn't provide useful error messages, so we do some
+    // sanity checks.
+    // Also check that the iters is not negative.
+    BM_CHECK(num_iters > 0 && !(num_iters == 0 && num_iters_str != "0"))
+        << "Malformed iters value passed to --benchmark_min_time: `" << value
+        << "`. Expected --benchmark_min_time=<integer>x.";
+
+    ret.tag = BenchTimeType::ITERS;
+    ret.iters = num_iters;
+    return ret;
+  }
+
+  std::string min_time_str;
+  if (value.back() != 's') {
+    BM_VLOG(2) << "Value passed to --benchmark_min_time should have a suffix. "
+                  "Eg., `30s` for 30-seconds.";
+    min_time_str = value;
+  } else {
+    min_time_str = value.substr(0, value.length() - 1);
+  }
+  double min_time = std::atof(min_time_str.c_str());
+
+  BM_CHECK(min_time > 0 &&
+           !(min_time == 0 && (min_time_str != "0" || min_time_str != "0.0")))
+      << "Malformed seconds value passed to --benchmark_min_time: `" << value
+      << "`. Expected --benchmark_min_time=<float>x.";
+
+  ret.tag = BenchTimeType::TIME;
+  ret.time = min_time;
+
+  return ret;
+}
+
+double GetMinTime(const benchmark::internal::BenchmarkInstance& b) {
+  if (!IsZero(b.min_time())) return b.min_time();
+  BenchTimeType iters_or_time = ParseBenchMinTime(FLAGS_benchmark_min_time);
+  // If the flag was used to specify number of iters, then return 0 for time.
+  if (iters_or_time.tag == BenchTimeType::ITERS) return 0;
+
+  return iters_or_time.time;
+}
+
+bool BenchMinTimeHasIters() {
+  return !FLAGS_benchmark_min_time.empty() &&
+         FLAGS_benchmark_min_time.back() == 'x';
+}
+
+IterationCount GetIters(const benchmark::internal::BenchmarkInstance& b) {
+  if (b.iterations() != 0) return b.iterations();
+
+  // We've already checked that this flag is currently used to pass
+  // iters but do a sanity check anyway.
+  BenchTimeType parsed = ParseBenchMinTime(FLAGS_benchmark_min_time);
+  assert(parsed.tag == BenchTimeType::ITERS);
+  return parsed.iters;
+}
+
 }  // end namespace
 
 BenchmarkRunner::BenchmarkRunner(
     const benchmark::internal::BenchmarkInstance& b_,
     BenchmarkReporter::PerFamilyRunReports* reports_for_family_)
     : b(b_),
       reports_for_family(reports_for_family_),
-      min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time),
+      min_time(GetMinTime(b_)),
       min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0)
                           ? b.min_warmup_time()
                           : FLAGS_benchmark_min_warmup_time),
       warmup_done(!(min_warmup_time > 0.0)),
       repeats(b.repetitions() != 0 ? b.repetitions()
                                    : FLAGS_benchmark_repetitions),
-      has_explicit_iteration_count(b.iterations() != 0),
+      has_explicit_iteration_count(b.iterations() != 0 ||
+                                   BenchMinTimeHasIters()),
       pool(b.threads() - 1),
-      iters(has_explicit_iteration_count ? b.iterations() : 1),
+      iters(has_explicit_iteration_count ? GetIters(b_) : 1),
       perf_counters_measurement(StrSplit(FLAGS_benchmark_perf_counters, ',')),
       perf_counters_measurement_ptr(perf_counters_measurement.IsValid()
                                         ? &perf_counters_measurement

src/benchmark_runner.h

@@ -25,7 +25,7 @@
 
 namespace benchmark {
 
-BM_DECLARE_double(benchmark_min_time);
+BM_DECLARE_string(benchmark_min_time);
 BM_DECLARE_double(benchmark_min_warmup_time);
 BM_DECLARE_int32(benchmark_repetitions);
 BM_DECLARE_bool(benchmark_report_aggregates_only);

test/BUILD

@@ -25,14 +25,17 @@ PER_SRC_COPTS = {
     "donotoptimize_test.cc": ["-O3"],
 }
 
-TEST_ARGS = ["--benchmark_min_time=0.01"]
+TEST_ARGS = ["--benchmark_min_time=0.01s"]
 
-PER_SRC_TEST_ARGS = ({
+PER_SRC_TEST_ARGS = {
     "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"],
     "repetitions_test.cc": [" --benchmark_repetitions=3"],
-    "spec_arg_test.cc" : ["--benchmark_filter=BM_NotChosen"],
-    "spec_arg_verbosity_test.cc" : ["--v=42"],
-})
+    "spec_arg_test.cc": ["--benchmark_filter=BM_NotChosen"],
+    "spec_arg_verbosity_test.cc": ["--v=42"],
+    "benchmark_min_time_flag_iters_test.cc": ["--benchmark_min_time=4x"],
+    # Backward compat test.
+    "benchmark_min_time_flag_time_test.cc": ["--benchmark_min_time=2"],
+}
 
 cc_library(
     name = "output_test_helper",
@@ -58,14 +61,14 @@ cc_library(
         copts = select({
             "//:windows": [],
             "//conditions:default": TEST_COPTS,
-        }) + PER_SRC_COPTS.get(test_src, []) ,
+        }) + PER_SRC_COPTS.get(test_src, []),
         deps = [
             ":output_test_helper",
             "//:benchmark",
             "//:benchmark_internal_headers",
             "@com_google_googletest//:gtest",
             "@com_google_googletest//:gtest_main",
-        ]
+        ],
         # FIXME: Add support for assembly tests to bazel.
         # See Issue #556
         # https://github.com/google/benchmark/issues/556
@@ -85,17 +88,17 @@ cc_test(
     size = "small",
     srcs = ["cxx03_test.cc"],
     copts = TEST_COPTS + ["-std=c++03"],
+    target_compatible_with = select({
+        "//:windows": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
     deps = [
         ":output_test_helper",
         "//:benchmark",
         "//:benchmark_internal_headers",
         "@com_google_googletest//:gtest",
         "@com_google_googletest//:gtest_main",
     ],
-    target_compatible_with = select({
-        "//:windows": ["@platforms//:incompatible"],
-        "//conditions:default": [],
-    })
 )
 
 cc_test(

test/benchmark_min_time_flag_iters_test.cc

@@ -0,0 +1,65 @@
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+
+// Tests that we can specify the number of iterations with
+// --benchmark_min_time=<NUM>x.
+namespace {
+
+class TestReporter : public benchmark::ConsoleReporter {
+ public:
+  virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
+    return ConsoleReporter::ReportContext(context);
+  };
+
+  virtual void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
+    assert(report.size() == 1);
+    iter_nums_.push_back(report[0].iterations);
+    ConsoleReporter::ReportRuns(report);
+  };
+
+  TestReporter() {}
+
+  virtual ~TestReporter() {}
+
+  const std::vector<int>& GetIters() const { return iter_nums_; }
+
+ private:
+  std::vector<int> iter_nums_;
+};
+
+}  // end namespace
+
+static void BM_MyBench(benchmark::State& state) {
+  for (auto s : state) {
+  }
+}
+BENCHMARK(BM_MyBench);
+
+int main(int argc, char** argv) {
+  // Verify that argv specify --benchmark_min_time=4x
+  bool found = false;
+  for (int i = 0; i < argc; ++i) {
+    if (strcmp("--benchmark_min_time=4x", argv[i]) == 0) {
+      found = true;
+      break;
+    }
+  }
+  assert(found);
+  benchmark::Initialize(&argc, argv);
+
+  TestReporter test_reporter;
+  const size_t returned_count =
+      benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench");
+  assert(returned_count == 1);
+
+  // Check the executed iters.
+  const std::vector<int> iters = test_reporter.GetIters();
+  assert(!iters.empty() && iters[0] == 4);
+  return 0;
+}

test/benchmark_min_time_flag_time_test.cc

@@ -0,0 +1,82 @@
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+
+// Tests that we can specify the min time with
+// --benchmark_min_time=<NUM> (no suffix needed) OR
+// --benchmark_min_time=<NUM>s
+namespace {
+
+class TestReporter : public benchmark::ConsoleReporter {
+ public:
+  virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE {
+    return ConsoleReporter::ReportContext(context);
+  };
+
+  virtual void ReportRuns(const std::vector<Run>& report) BENCHMARK_OVERRIDE {
+    assert(report.size() == 1);
+    min_times_.push_back(report[0].run_name.min_time);
+    ConsoleReporter::ReportRuns(report);
+  };
+
+  TestReporter() {}
+
+  virtual ~TestReporter() {}
+
+  const std::vector<std::string>& GetMinTimes() const { return min_times_; }
+
+ private:
+  std::vector<std::string> min_times_;
+};
+
+void DoTestHelper(int* argc, const char** argv, const std::string& expected) {
+  benchmark::Initialize(argc, const_cast<char**>(argv));
+
+  TestReporter test_reporter;
+  const size_t returned_count =
+      benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench");
+  assert(returned_count == 1);
+
+  // Check the min_time
+  const std::vector<std::string>& min_times = test_reporter.GetMinTimes();
+  if (min_times.empty()) std::cout << "**** min_times empty\n";
+  else {
+    std::cout << " *** EXPECTED = " << expected << "\n";
+    for (const std::string& v : min_times)
+      std::cout << "   ** v = " << v << "\n";
+  }
+  assert(!min_times.empty() && min_times[0] == expected);
+}
+
+}  // end namespace
+
+static void BM_MyBench(benchmark::State& state) {
+  for (auto s : state) {
+  }
+}
+BENCHMARK(BM_MyBench);
+
+int main(int argc, char** argv) {
+  // Make a fake argv and append the new --benchmark_min_time=<foo> to it.
+  int fake_argc = argc + 1;
+  const char** fake_argv = new const char*[fake_argc];
+
+  for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i];
+
+  const char* no_suffix = "--benchmark_min_time=4.0";
+  const char* with_suffix = "--benchmark_min_time=4.0s";
+  std::string expected = "min_time:4.0s";
+
+  fake_argv[argc] = no_suffix;
+  DoTestHelper(&fake_argc, fake_argv, expected);
+
+  fake_argv[argc] = with_suffix;
+  DoTestHelper(&fake_argc, fake_argv, expected);
+
+  return 0;
+}