Add --benchmark_report_aggregates_only={true|false} flag for better summary output. (#267)

README.md

@@ -391,6 +391,13 @@ The number of runs of each benchmark is specified globally by the
 `Repetitions` on the registered benchmark object. When a benchmark is run
 more than once the mean and standard deviation of the runs will be reported.
 
+Additionally the `--benchmark_report_aggregates_only={true|false}` flag or
+`ReportAggregatesOnly(bool)` function can be used to change how repeated tests
+are reported. By default the result of each repeated run is reported. When this
+option is 'true' only the mean and standard deviation of the runs is reported.
+Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides
+the value of the flag for that benchmark.
+
 ## Fixtures
 Fixture tests are created by
 first defining a type that derives from ::benchmark::Fixture and then

include/benchmark/benchmark_api.h

@@ -522,6 +522,11 @@ public:
   // REQUIRES: `n > 0`
   Benchmark* Repetitions(int n);
 
+  // Specify if each repetition of the benchmark should be reported separately
+  // or if only the final statistics should be reported. If the benchmark
+  // is not repeated then the single result is always reported.
+  Benchmark* ReportAggregatesOnly(bool v = true);
+
   // If a particular benchmark is I/O bound, runs multiple threads internally or
   // if for some reason CPU timings are not representative, call this method. If
   // called, the elapsed time will be used to control how many iterations are

src/benchmark.cc

@@ -66,6 +66,11 @@ DEFINE_int32(benchmark_repetitions, 1,
              "The number of runs of each benchmark. If greater than 1, the "
              "mean and standard deviation of the runs will be reported.");
 
+DEFINE_bool(benchmark_report_aggregates_only, false,
+            "Report the result of each benchmark repetitions. When 'true' is "
+            "specified only the mean, standard deviation, and other statistics "
+            "are reported for repeated benchmarks.");
+
 DEFINE_string(benchmark_format, "console",
               "The format to use for console output. Valid values are "
               "'console', 'json', or 'csv'.");
@@ -311,10 +316,17 @@ static std::unique_ptr<TimerManager> timer_manager = nullptr;
 
 namespace internal {
 
+enum ReportMode : unsigned {
+    RM_Unspecified, // The mode has not been manually specified
+    RM_Default,     // The mode is user-specified as default.
+    RM_ReportAggregatesOnly
+};
+
 // Information kept per benchmark we may want to run
 struct Benchmark::Instance {
   std::string      name;
   Benchmark*       benchmark;
+  ReportMode       report_mode;
   std::vector<int> arg;
   TimeUnit         time_unit;
   int              range_multiplier;
@@ -364,6 +376,7 @@ public:
   void RangeMultiplier(int multiplier);
   void MinTime(double n);
   void Repetitions(int n);
+  void ReportAggregatesOnly(bool v);
   void UseRealTime();
   void UseManualTime();
   void Complexity(BigO complexity);
@@ -381,6 +394,7 @@ private:
   friend class BenchmarkFamilies;
 
   std::string name_;
+  ReportMode report_mode_;
   std::vector< std::vector<int> > args_;  // Args for all benchmark runs
   TimeUnit time_unit_;
   int range_multiplier_;
@@ -443,6 +457,7 @@ bool BenchmarkFamilies::FindBenchmarks(
         Benchmark::Instance instance;
         instance.name = family->name_;
         instance.benchmark = bench_family.get();
+        instance.report_mode = family->report_mode_;
         instance.arg = args;
         instance.time_unit = family->time_unit_;
         instance.range_multiplier = family->range_multiplier_;
@@ -488,7 +503,7 @@ bool BenchmarkFamilies::FindBenchmarks(
 }
 
 BenchmarkImp::BenchmarkImp(const char* name)
-    : name_(name), time_unit_(kNanosecond),
+    : name_(name), report_mode_(RM_Unspecified), time_unit_(kNanosecond),
       range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0),
       use_real_time_(false), use_manual_time_(false),
       complexity_(oNone) {
@@ -575,6 +590,10 @@ void BenchmarkImp::Repetitions(int n) {
   repetitions_ = n;
 }
 
+void BenchmarkImp::ReportAggregatesOnly(bool value) {
+  report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default;
+}
+
 void BenchmarkImp::UseRealTime() {
   CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously.";
   use_real_time_ = true;
@@ -703,6 +722,11 @@ Benchmark* Benchmark::Repetitions(int t) {
   return this;
 }
 
+Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
+  imp_->ReportAggregatesOnly(value);
+  return this;
+}
+
 Benchmark* Benchmark::MinTime(double t) {
   imp_->MinTime(t);
   return this;
@@ -779,7 +803,8 @@ std::vector<BenchmarkReporter::Run>
 RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
              std::vector<BenchmarkReporter::Run>* complexity_reports)
   EXCLUDES(GetBenchmarkLock()) {
-   std::vector<BenchmarkReporter::Run> reports; // return value
+  std::vector<BenchmarkReporter::Run> reports; // return value
+
   size_t iters = 1;
 
   std::vector<std::thread> pool;
@@ -788,6 +813,10 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
 
   const int repeats = b.repetitions != 0 ? b.repetitions
                                          : FLAGS_benchmark_repetitions;
+  const bool report_aggregates_only = repeats != 1 &&
+      (b.report_mode == internal::RM_Unspecified
+        ? FLAGS_benchmark_report_aggregates_only
+        : b.report_mode == internal::RM_ReportAggregatesOnly);
   for (int i = 0; i < repeats; i++) {
     std::string mem;
     for (;;) {
@@ -914,22 +943,21 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
       iters = static_cast<int>(next_iters + 0.5);
     }
   }
-  std::vector<BenchmarkReporter::Run> additional_run_stats = ComputeStats(reports);
-  reports.insert(reports.end(), additional_run_stats.begin(),
-                 additional_run_stats.end());
-
-  if((b.complexity != oNone) && b.last_benchmark_instance) {
-    additional_run_stats = ComputeBigO(*complexity_reports);
-    reports.insert(reports.end(), additional_run_stats.begin(),
-                   additional_run_stats.end());
-    complexity_reports->clear();
-  }
-
   if (b.multithreaded) {
     for (std::thread& thread : pool)
       thread.join();
   }
+  // Calculate additional statistics
+  auto stat_reports = ComputeStats(reports);
+  if((b.complexity != oNone) && b.last_benchmark_instance) {
+    auto additional_run_stats = ComputeBigO(*complexity_reports);
+    stat_reports.insert(stat_reports.end(), additional_run_stats.begin(),
+                   additional_run_stats.end());
+    complexity_reports->clear();
+  }
 
+  if (report_aggregates_only) reports.clear();
+  reports.insert(reports.end(), stat_reports.begin(), stat_reports.end());
   return reports;
 }
 
@@ -1117,6 +1145,7 @@ void PrintUsageAndExit() {
           "          [--benchmark_filter=<regex>]\n"
           "          [--benchmark_min_time=<min_time>]\n"
           "          [--benchmark_repetitions=<num_repetitions>]\n"
+          "          [--benchmark_report_aggregates_only={true|false}\n"
           "          [--benchmark_format=<console|json|csv>]\n"
           "          [--benchmark_out=<filename>]\n"
           "          [--benchmark_out_format=<json|console|csv>]\n"
@@ -1137,6 +1166,8 @@ void ParseCommandLineFlags(int* argc, char** argv) {
                         &FLAGS_benchmark_min_time) ||
         ParseInt32Flag(argv[i], "benchmark_repetitions",
                        &FLAGS_benchmark_repetitions) ||
+        ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
+                       &FLAGS_benchmark_report_aggregates_only) ||
         ParseStringFlag(argv[i], "benchmark_format",
                         &FLAGS_benchmark_format) ||
         ParseStringFlag(argv[i], "benchmark_out",

test/reporter_output_test.cc

@@ -6,6 +6,7 @@
 #include <cassert>
 #include <cstring>
 #include <iostream>
+#include <memory>
 #include <sstream>
 #include <vector>
 #include <utility>
@@ -18,35 +19,58 @@ namespace {
 
 enum MatchRules {
   MR_Default, // Skip non-matching lines until a match is found.
-  MR_Next    // Match must occur on the next line.
+  MR_Next,    // Match must occur on the next line.
+  MR_Not      // No line between the current position and the next match matches
+              // the regex
 };
 
 struct TestCase {
-  std::string regex;
+  std::string regex_str;
   int match_rule;
+  std::shared_ptr<benchmark::Regex> regex;
 
-  TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
+  TestCase(std::string re, int rule = MR_Default)
-
+      : regex_str(re), match_rule(rule), regex(std::make_shared<benchmark::Regex>()) {
-  void Check(std::stringstream& remaining_output) const {
-    benchmark::Regex r;
     std::string err_str;
-    r.Init(regex, &err_str);
+    regex->Init(regex_str, &err_str);
-    CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
+    CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\""
                            << " got Error: " << err_str;
+  }
 
+  void Check(std::stringstream& remaining_output,
+             std::vector<TestCase>& not_checks) const {
     std::string line;
     while (remaining_output.eof() == false) {
         CHECK(remaining_output.good());
         std::getline(remaining_output, line);
-        if (r.Match(line)) return;
+        for (auto& NC : not_checks) {
+            CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
+                                          << line << "\" for MR_Not regex \""
+                                          << NC.regex_str << "\"";
+        }
+        if (regex->Match(line)) return;
         CHECK(match_rule != MR_Next) << "Expected line \"" << line
-                                     << "\" to match regex \"" << regex << "\"";
+                                     << "\" to match regex \"" << regex_str << "\"";
     }
 
     CHECK(remaining_output.eof() == false)
-        << "End of output reached before match for regex \"" << regex
+        << "End of output reached before match for regex \"" << regex_str
         << "\" was found";
   }
+
+  static void CheckCases(std::vector<TestCase> const& checks,
+                         std::stringstream& output) {
+      std::vector<TestCase> not_checks;
+      for (size_t i=0; i < checks.size(); ++i) {
+          const auto& TC = checks[i];
+          if (TC.match_rule == MR_Not) {
+              not_checks.push_back(TC);
+              continue;
+          }
+          TC.Check(output, not_checks);
+          not_checks.clear();
+      }
+  }
 };
 
 std::vector<TestCase> ConsoleOutputTests;
@@ -114,8 +138,6 @@ std::string join(First f, Args&&... args) {
     return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
 }
 
-
-
 std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
 
 }  // end namespace
@@ -199,6 +221,68 @@ ADD_CASES(&ConsoleOutputTests, {
 });
 
 
+// ========================================================================= //
+// ----------------------- Testing Aggregate Output ------------------------ //
+// ========================================================================= //
+
+// Test that non-aggregate data is printed by default
+void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_Repeat)->Repetitions(3);
+ADD_CASES(&ConsoleOutputTests, {
+    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+    {"\"name\": \"BM_Repeat/repeats:3\",$"},
+    {"\"name\": \"BM_Repeat/repeats:3\",$"},
+    {"\"name\": \"BM_Repeat/repeats:3\",$"},
+    {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
+    {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
+// Test that a non-repeated test still prints non-aggregate results even when
+// only-aggregate reports have been requested
+void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
+ADD_CASES(&ConsoleOutputTests, {
+    {"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+    {"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+    {"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
+// Test that non-aggregate data is not reported
+void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
+ADD_CASES(&ConsoleOutputTests, {
+    {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+    {"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+    {"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+    {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+    {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
+    {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+    {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+    {"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+    {"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
 // ========================================================================= //
 // --------------------------- TEST CASES END ------------------------------ //
 // ========================================================================= //
@@ -244,10 +328,8 @@ int main(int argc, char* argv[]) {
       std::cerr << rep_test.err_stream.str();
       std::cout << rep_test.out_stream.str();
 
-      for (const auto& TC : rep_test.error_cases)
+      TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream);
-        TC.Check(rep_test.err_stream);
+      TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream);
-      for (const auto& TC : rep_test.output_cases)
-        TC.Check(rep_test.out_stream);
 
       std::cout << "\n";
   }