mirror of
https://github.com/google/benchmark.git
synced 2025-01-27 04:10:16 +08:00
Add --benchmark_report_aggregates_only={true|false} flag for better summary output. (#267)
This commit is contained in:
Eric
GitHub
parent
c9d747e5e6
commit
a11fb69c89
@ -391,6 +391,13 @@ The number of runs of each benchmark is specified globally by the
|
||||
`Repetitions` on the registered benchmark object. When a benchmark is run
|
||||
more than once the mean and standard deviation of the runs will be reported.
|
||||
|
||||
Additionally the `--benchmark_report_aggregates_only={true|false}` flag or
|
||||
`ReportAggregatesOnly(bool)` function can be used to change how repeated tests
|
||||
are reported. By default the result of each repeated run is reported. When this
|
||||
option is 'true' only the mean and standard deviation of the runs is reported.
|
||||
Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides
|
||||
the value of the flag for that benchmark.
|
||||
|
||||
## Fixtures
|
||||
Fixture tests are created by
|
||||
first defining a type that derives from ::benchmark::Fixture and then
|
||||
|
||||
@ -522,6 +522,11 @@ public:
|
||||
// REQUIRES: `n > 0`
|
||||
Benchmark* Repetitions(int n);
|
||||
|
||||
// Specify if each repetition of the benchmark should be reported separately
|
||||
// or if only the final statistics should be reported. If the benchmark
|
||||
// is not repeated then the single result is always reported.
|
||||
Benchmark* ReportAggregatesOnly(bool v = true);
|
||||
|
||||
// If a particular benchmark is I/O bound, runs multiple threads internally or
|
||||
// if for some reason CPU timings are not representative, call this method. If
|
||||
// called, the elapsed time will be used to control how many iterations are
|
||||
|
||||
@ -66,6 +66,11 @@ DEFINE_int32(benchmark_repetitions, 1,
|
||||
"The number of runs of each benchmark. If greater than 1, the "
|
||||
"mean and standard deviation of the runs will be reported.");
|
||||
|
||||
DEFINE_bool(benchmark_report_aggregates_only, false,
|
||||
"Report the result of each benchmark repetitions. When 'true' is "
|
||||
"specified only the mean, standard deviation, and other statistics "
|
||||
"are reported for repeated benchmarks.");
|
||||
|
||||
DEFINE_string(benchmark_format, "console",
|
||||
"The format to use for console output. Valid values are "
|
||||
"'console', 'json', or 'csv'.");
|
||||
@ -311,10 +316,17 @@ static std::unique_ptr<TimerManager> timer_manager = nullptr;
|
||||
|
||||
namespace internal {
|
||||
|
||||
enum ReportMode : unsigned {
|
||||
RM_Unspecified, // The mode has not been manually specified
|
||||
RM_Default, // The mode is user-specified as default.
|
||||
RM_ReportAggregatesOnly
|
||||
};
|
||||
|
||||
// Information kept per benchmark we may want to run
|
||||
struct Benchmark::Instance {
|
||||
std::string name;
|
||||
Benchmark* benchmark;
|
||||
ReportMode report_mode;
|
||||
std::vector<int> arg;
|
||||
TimeUnit time_unit;
|
||||
int range_multiplier;
|
||||
@ -364,6 +376,7 @@ public:
|
||||
void RangeMultiplier(int multiplier);
|
||||
void MinTime(double n);
|
||||
void Repetitions(int n);
|
||||
void ReportAggregatesOnly(bool v);
|
||||
void UseRealTime();
|
||||
void UseManualTime();
|
||||
void Complexity(BigO complexity);
|
||||
@ -381,6 +394,7 @@ private:
|
||||
friend class BenchmarkFamilies;
|
||||
|
||||
std::string name_;
|
||||
ReportMode report_mode_;
|
||||
std::vector< std::vector<int> > args_; // Args for all benchmark runs
|
||||
TimeUnit time_unit_;
|
||||
int range_multiplier_;
|
||||
@ -443,6 +457,7 @@ bool BenchmarkFamilies::FindBenchmarks(
|
||||
Benchmark::Instance instance;
|
||||
instance.name = family->name_;
|
||||
instance.benchmark = bench_family.get();
|
||||
instance.report_mode = family->report_mode_;
|
||||
instance.arg = args;
|
||||
instance.time_unit = family->time_unit_;
|
||||
instance.range_multiplier = family->range_multiplier_;
|
||||
@ -488,7 +503,7 @@ bool BenchmarkFamilies::FindBenchmarks(
|
||||
}
|
||||
|
||||
BenchmarkImp::BenchmarkImp(const char* name)
|
||||
: name_(name), time_unit_(kNanosecond),
|
||||
: name_(name), report_mode_(RM_Unspecified), time_unit_(kNanosecond),
|
||||
range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0),
|
||||
use_real_time_(false), use_manual_time_(false),
|
||||
complexity_(oNone) {
|
||||
@ -575,6 +590,10 @@ void BenchmarkImp::Repetitions(int n) {
|
||||
repetitions_ = n;
|
||||
}
|
||||
|
||||
void BenchmarkImp::ReportAggregatesOnly(bool value) {
|
||||
report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default;
|
||||
}
|
||||
|
||||
void BenchmarkImp::UseRealTime() {
|
||||
CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously.";
|
||||
use_real_time_ = true;
|
||||
@ -703,6 +722,11 @@ Benchmark* Benchmark::Repetitions(int t) {
|
||||
return this;
|
||||
}
|
||||
|
||||
Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
|
||||
imp_->ReportAggregatesOnly(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
Benchmark* Benchmark::MinTime(double t) {
|
||||
imp_->MinTime(t);
|
||||
return this;
|
||||
@ -779,7 +803,8 @@ std::vector<BenchmarkReporter::Run>
|
||||
RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
|
||||
std::vector<BenchmarkReporter::Run>* complexity_reports)
|
||||
EXCLUDES(GetBenchmarkLock()) {
|
||||
std::vector<BenchmarkReporter::Run> reports; // return value
|
||||
std::vector<BenchmarkReporter::Run> reports; // return value
|
||||
|
||||
size_t iters = 1;
|
||||
|
||||
std::vector<std::thread> pool;
|
||||
@ -788,6 +813,10 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
|
||||
|
||||
const int repeats = b.repetitions != 0 ? b.repetitions
|
||||
: FLAGS_benchmark_repetitions;
|
||||
const bool report_aggregates_only = repeats != 1 &&
|
||||
(b.report_mode == internal::RM_Unspecified
|
||||
? FLAGS_benchmark_report_aggregates_only
|
||||
: b.report_mode == internal::RM_ReportAggregatesOnly);
|
||||
for (int i = 0; i < repeats; i++) {
|
||||
std::string mem;
|
||||
for (;;) {
|
||||
@ -914,22 +943,21 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
|
||||
iters = static_cast<int>(next_iters + 0.5);
|
||||
}
|
||||
}
|
||||
std::vector<BenchmarkReporter::Run> additional_run_stats = ComputeStats(reports);
|
||||
reports.insert(reports.end(), additional_run_stats.begin(),
|
||||
additional_run_stats.end());
|
||||
|
||||
if((b.complexity != oNone) && b.last_benchmark_instance) {
|
||||
additional_run_stats = ComputeBigO(*complexity_reports);
|
||||
reports.insert(reports.end(), additional_run_stats.begin(),
|
||||
additional_run_stats.end());
|
||||
complexity_reports->clear();
|
||||
}
|
||||
|
||||
if (b.multithreaded) {
|
||||
for (std::thread& thread : pool)
|
||||
thread.join();
|
||||
}
|
||||
// Calculate additional statistics
|
||||
auto stat_reports = ComputeStats(reports);
|
||||
if((b.complexity != oNone) && b.last_benchmark_instance) {
|
||||
auto additional_run_stats = ComputeBigO(*complexity_reports);
|
||||
stat_reports.insert(stat_reports.end(), additional_run_stats.begin(),
|
||||
additional_run_stats.end());
|
||||
complexity_reports->clear();
|
||||
}
|
||||
|
||||
if (report_aggregates_only) reports.clear();
|
||||
reports.insert(reports.end(), stat_reports.begin(), stat_reports.end());
|
||||
return reports;
|
||||
}
|
||||
|
||||
@ -1117,6 +1145,7 @@ void PrintUsageAndExit() {
|
||||
" [--benchmark_filter=<regex>]\n"
|
||||
" [--benchmark_min_time=<min_time>]\n"
|
||||
" [--benchmark_repetitions=<num_repetitions>]\n"
|
||||
" [--benchmark_report_aggregates_only={true|false}\n"
|
||||
" [--benchmark_format=<console|json|csv>]\n"
|
||||
" [--benchmark_out=<filename>]\n"
|
||||
" [--benchmark_out_format=<json|console|csv>]\n"
|
||||
@ -1137,6 +1166,8 @@ void ParseCommandLineFlags(int* argc, char** argv) {
|
||||
&FLAGS_benchmark_min_time) ||
|
||||
ParseInt32Flag(argv[i], "benchmark_repetitions",
|
||||
&FLAGS_benchmark_repetitions) ||
|
||||
ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
|
||||
&FLAGS_benchmark_report_aggregates_only) ||
|
||||
ParseStringFlag(argv[i], "benchmark_format",
|
||||
&FLAGS_benchmark_format) ||
|
||||
ParseStringFlag(argv[i], "benchmark_out",
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
@ -18,35 +19,58 @@ namespace {
|
||||
|
||||
enum MatchRules {
|
||||
MR_Default, // Skip non-matching lines until a match is found.
|
||||
MR_Next // Match must occur on the next line.
|
||||
MR_Next, // Match must occur on the next line.
|
||||
MR_Not // No line between the current position and the next match matches
|
||||
// the regex
|
||||
};
|
||||
|
||||
struct TestCase {
|
||||
std::string regex;
|
||||
std::string regex_str;
|
||||
int match_rule;
|
||||
std::shared_ptr<benchmark::Regex> regex;
|
||||
|
||||
TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
|
||||
|
||||
void Check(std::stringstream& remaining_output) const {
|
||||
benchmark::Regex r;
|
||||
TestCase(std::string re, int rule = MR_Default)
|
||||
: regex_str(re), match_rule(rule), regex(std::make_shared<benchmark::Regex>()) {
|
||||
std::string err_str;
|
||||
r.Init(regex, &err_str);
|
||||
CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
|
||||
regex->Init(regex_str, &err_str);
|
||||
CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\""
|
||||
<< " got Error: " << err_str;
|
||||
}
|
||||
|
||||
void Check(std::stringstream& remaining_output,
|
||||
std::vector<TestCase>& not_checks) const {
|
||||
std::string line;
|
||||
while (remaining_output.eof() == false) {
|
||||
CHECK(remaining_output.good());
|
||||
std::getline(remaining_output, line);
|
||||
if (r.Match(line)) return;
|
||||
for (auto& NC : not_checks) {
|
||||
CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
|
||||
<< line << "\" for MR_Not regex \""
|
||||
<< NC.regex_str << "\"";
|
||||
}
|
||||
if (regex->Match(line)) return;
|
||||
CHECK(match_rule != MR_Next) << "Expected line \"" << line
|
||||
<< "\" to match regex \"" << regex << "\"";
|
||||
<< "\" to match regex \"" << regex_str << "\"";
|
||||
}
|
||||
|
||||
CHECK(remaining_output.eof() == false)
|
||||
<< "End of output reached before match for regex \"" << regex
|
||||
<< "End of output reached before match for regex \"" << regex_str
|
||||
<< "\" was found";
|
||||
}
|
||||
|
||||
static void CheckCases(std::vector<TestCase> const& checks,
|
||||
std::stringstream& output) {
|
||||
std::vector<TestCase> not_checks;
|
||||
for (size_t i=0; i < checks.size(); ++i) {
|
||||
const auto& TC = checks[i];
|
||||
if (TC.match_rule == MR_Not) {
|
||||
not_checks.push_back(TC);
|
||||
continue;
|
||||
}
|
||||
TC.Check(output, not_checks);
|
||||
not_checks.clear();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<TestCase> ConsoleOutputTests;
|
||||
@ -114,8 +138,6 @@ std::string join(First f, Args&&... args) {
|
||||
return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
|
||||
|
||||
std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
||||
|
||||
} // end namespace
|
||||
@ -199,6 +221,68 @@ ADD_CASES(&ConsoleOutputTests, {
|
||||
});
|
||||
|
||||
|
||||
// ========================================================================= //
|
||||
// ----------------------- Testing Aggregate Output ------------------------ //
|
||||
// ========================================================================= //
|
||||
|
||||
// Test that non-aggregate data is printed by default
|
||||
void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_Repeat)->Repetitions(3);
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
});
|
||||
|
||||
// Test that a non-repeated test still prints non-aggregate results even when
|
||||
// only-aggregate reports have been requested
|
||||
void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
});
|
||||
|
||||
// Test that non-aggregate data is not reported
|
||||
void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} }
|
||||
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
|
||||
ADD_CASES(&ConsoleOutputTests, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
|
||||
});
|
||||
ADD_CASES(&JSONOutputTests, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}
|
||||
});
|
||||
ADD_CASES(&CSVOutputTests, {
|
||||
{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
|
||||
});
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- TEST CASES END ------------------------------ //
|
||||
// ========================================================================= //
|
||||
@ -244,10 +328,8 @@ int main(int argc, char* argv[]) {
|
||||
std::cerr << rep_test.err_stream.str();
|
||||
std::cout << rep_test.out_stream.str();
|
||||
|
||||
for (const auto& TC : rep_test.error_cases)
|
||||
TC.Check(rep_test.err_stream);
|
||||
for (const auto& TC : rep_test.output_cases)
|
||||
TC.Check(rep_test.out_stream);
|
||||
TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream);
|
||||
TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream);
|
||||
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user