diff --git a/README.md b/README.md index 8cf853cb..0c081ff8 100644 --- a/README.md +++ b/README.md @@ -391,6 +391,13 @@ The number of runs of each benchmark is specified globally by the `Repetitions` on the registered benchmark object. When a benchmark is run more than once the mean and standard deviation of the runs will be reported. +Additionally the `--benchmark_report_aggregates_only={true|false}` flag or +`ReportAggregatesOnly(bool)` function can be used to change how repeated tests +are reported. By default the result of each repeated run is reported. When this +option is 'true' only the mean and standard deviation of the runs is reported. +Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides +the value of the flag for that benchmark. + ## Fixtures Fixture tests are created by first defining a type that derives from ::benchmark::Fixture and then diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 1a481ac4..ddfcb014 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -522,6 +522,11 @@ public: // REQUIRES: `n > 0` Benchmark* Repetitions(int n); + // Specify if each repetition of the benchmark should be reported separately + // or if only the final statistics should be reported. If the benchmark + // is not repeated then the single result is always reported. + Benchmark* ReportAggregatesOnly(bool v = true); + // If a particular benchmark is I/O bound, runs multiple threads internally or // if for some reason CPU timings are not representative, call this method. If // called, the elapsed time will be used to control how many iterations are diff --git a/src/benchmark.cc b/src/benchmark.cc index 515abd76..3a199e50 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -66,6 +66,11 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); +DEFINE_bool(benchmark_report_aggregates_only, false, + "Report the result of each benchmark repetitions. When 'true' is " + "specified only the mean, standard deviation, and other statistics " + "are reported for repeated benchmarks."); + DEFINE_string(benchmark_format, "console", "The format to use for console output. Valid values are " "'console', 'json', or 'csv'."); @@ -311,10 +316,17 @@ static std::unique_ptr timer_manager = nullptr; namespace internal { +enum ReportMode : unsigned { + RM_Unspecified, // The mode has not been manually specified + RM_Default, // The mode is user-specified as default. + RM_ReportAggregatesOnly +}; + // Information kept per benchmark we may want to run struct Benchmark::Instance { std::string name; Benchmark* benchmark; + ReportMode report_mode; std::vector arg; TimeUnit time_unit; int range_multiplier; @@ -364,6 +376,7 @@ public: void RangeMultiplier(int multiplier); void MinTime(double n); void Repetitions(int n); + void ReportAggregatesOnly(bool v); void UseRealTime(); void UseManualTime(); void Complexity(BigO complexity); @@ -381,6 +394,7 @@ private: friend class BenchmarkFamilies; std::string name_; + ReportMode report_mode_; std::vector< std::vector > args_; // Args for all benchmark runs TimeUnit time_unit_; int range_multiplier_; @@ -443,6 +457,7 @@ bool BenchmarkFamilies::FindBenchmarks( Benchmark::Instance instance; instance.name = family->name_; instance.benchmark = bench_family.get(); + instance.report_mode = family->report_mode_; instance.arg = args; instance.time_unit = family->time_unit_; instance.range_multiplier = family->range_multiplier_; @@ -488,7 +503,7 @@ bool BenchmarkFamilies::FindBenchmarks( } BenchmarkImp::BenchmarkImp(const char* name) - : name_(name), time_unit_(kNanosecond), + : name_(name), report_mode_(RM_Unspecified), time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0), use_real_time_(false), use_manual_time_(false), complexity_(oNone) { @@ -575,6 +590,10 @@ void BenchmarkImp::Repetitions(int n) { repetitions_ = n; } +void BenchmarkImp::ReportAggregatesOnly(bool value) { + report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default; +} + void BenchmarkImp::UseRealTime() { CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; use_real_time_ = true; @@ -703,6 +722,11 @@ Benchmark* Benchmark::Repetitions(int t) { return this; } +Benchmark* Benchmark::ReportAggregatesOnly(bool value) { + imp_->ReportAggregatesOnly(value); + return this; +} + Benchmark* Benchmark::MinTime(double t) { imp_->MinTime(t); return this; @@ -779,7 +803,8 @@ std::vector RunBenchmark(const benchmark::internal::Benchmark::Instance& b, std::vector* complexity_reports) EXCLUDES(GetBenchmarkLock()) { - std::vector reports; // return value + std::vector reports; // return value + size_t iters = 1; std::vector pool; @@ -788,6 +813,10 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b, const int repeats = b.repetitions != 0 ? b.repetitions : FLAGS_benchmark_repetitions; + const bool report_aggregates_only = repeats != 1 && + (b.report_mode == internal::RM_Unspecified + ? FLAGS_benchmark_report_aggregates_only + : b.report_mode == internal::RM_ReportAggregatesOnly); for (int i = 0; i < repeats; i++) { std::string mem; for (;;) { @@ -914,22 +943,21 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b, iters = static_cast(next_iters + 0.5); } } - std::vector additional_run_stats = ComputeStats(reports); - reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); - - if((b.complexity != oNone) && b.last_benchmark_instance) { - additional_run_stats = ComputeBigO(*complexity_reports); - reports.insert(reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); - complexity_reports->clear(); - } - if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); } + // Calculate additional statistics + auto stat_reports = ComputeStats(reports); + if((b.complexity != oNone) && b.last_benchmark_instance) { + auto additional_run_stats = ComputeBigO(*complexity_reports); + stat_reports.insert(stat_reports.end(), additional_run_stats.begin(), + additional_run_stats.end()); + complexity_reports->clear(); + } + if (report_aggregates_only) reports.clear(); + reports.insert(reports.end(), stat_reports.begin(), stat_reports.end()); return reports; } @@ -1117,6 +1145,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_report_aggregates_only={true|false}\n" " [--benchmark_format=]\n" " [--benchmark_out=]\n" " [--benchmark_out_format=]\n" @@ -1137,6 +1166,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_min_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", + &FLAGS_benchmark_report_aggregates_only) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseStringFlag(argv[i], "benchmark_out", diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index e580008e..fc71f27c 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -18,35 +19,58 @@ namespace { enum MatchRules { MR_Default, // Skip non-matching lines until a match is found. - MR_Next // Match must occur on the next line. + MR_Next, // Match must occur on the next line. + MR_Not // No line between the current position and the next match matches + // the regex }; struct TestCase { - std::string regex; + std::string regex_str; int match_rule; + std::shared_ptr regex; - TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {} - - void Check(std::stringstream& remaining_output) const { - benchmark::Regex r; + TestCase(std::string re, int rule = MR_Default) + : regex_str(re), match_rule(rule), regex(std::make_shared()) { std::string err_str; - r.Init(regex, &err_str); - CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" + regex->Init(regex_str, &err_str); + CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\"" << " got Error: " << err_str; + } + void Check(std::stringstream& remaining_output, + std::vector& not_checks) const { std::string line; while (remaining_output.eof() == false) { CHECK(remaining_output.good()); std::getline(remaining_output, line); - if (r.Match(line)) return; + for (auto& NC : not_checks) { + CHECK(!NC.regex->Match(line)) << "Unexpected match for line \"" + << line << "\" for MR_Not regex \"" + << NC.regex_str << "\""; + } + if (regex->Match(line)) return; CHECK(match_rule != MR_Next) << "Expected line \"" << line - << "\" to match regex \"" << regex << "\""; + << "\" to match regex \"" << regex_str << "\""; } CHECK(remaining_output.eof() == false) - << "End of output reached before match for regex \"" << regex + << "End of output reached before match for regex \"" << regex_str << "\" was found"; } + + static void CheckCases(std::vector const& checks, + std::stringstream& output) { + std::vector not_checks; + for (size_t i=0; i < checks.size(); ++i) { + const auto& TC = checks[i]; + if (TC.match_rule == MR_Not) { + not_checks.push_back(TC); + continue; + } + TC.Check(output, not_checks); + not_checks.clear(); + } + } }; std::vector ConsoleOutputTests; @@ -114,8 +138,6 @@ std::string join(First f, Args&&... args) { return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); } - - std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; } // end namespace @@ -199,6 +221,68 @@ ADD_CASES(&ConsoleOutputTests, { }); +// ========================================================================= // +// ----------------------- Testing Aggregate Output ------------------------ // +// ========================================================================= // + +// Test that non-aggregate data is printed by default +void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} } +BENCHMARK(BM_Repeat)->Repetitions(3); +ADD_CASES(&ConsoleOutputTests, { + {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}, + {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}, + {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}, + {"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}, + {"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"} +}); +ADD_CASES(&CSVOutputTests, { + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); + +// Test that a non-repeated test still prints non-aggregate results even when +// only-aggregate reports have been requested +void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} } +BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); +ADD_CASES(&ConsoleOutputTests, { + {"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"} +}); +ADD_CASES(&JSONOutputTests, { + {"\"name\": \"BM_RepeatOnce/repeats:1\",$"} +}); +ADD_CASES(&CSVOutputTests, { + {"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); + +// Test that non-aggregate data is not reported +void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} } +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); +ADD_CASES(&ConsoleOutputTests, { + {".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}, + {"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"} +}); +ADD_CASES(&JSONOutputTests, { + {".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"} +}); +ADD_CASES(&CSVOutputTests, { + {".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // @@ -244,10 +328,8 @@ int main(int argc, char* argv[]) { std::cerr << rep_test.err_stream.str(); std::cout << rep_test.out_stream.str(); - for (const auto& TC : rep_test.error_cases) - TC.Check(rep_test.err_stream); - for (const auto& TC : rep_test.output_cases) - TC.Check(rep_test.out_stream); + TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream); + TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream); std::cout << "\n"; }