mirror of
https://github.com/google/benchmark.git
synced 2025-03-14 03:10:22 +08:00
Aggregates: use non-aggregate count as iteration count. (#706)
It is incorrect to say that an aggregate is computed over run's iterations, because those iterations already got averaged. Similarly, if there are N repetitions with 1 iterations each, an aggregate will be computed over N measurements, not 1. Thus it is best to simply use the count of separate reports. Fixes #586.
This commit is contained in:
Roman Lebedev
GitHub
parent
99d1356c04
commit
507c06e636
@ -141,6 +141,9 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
|
||||
}
|
||||
}
|
||||
|
||||
const double iteration_rescale_factor =
|
||||
double(reports.size()) / double(run_iterations);
|
||||
|
||||
for (const auto& Stat : *reports[0].statistics) {
|
||||
// Get the data from the accumulator to BenchmarkReporter::Run's.
|
||||
Run data;
|
||||
@ -148,15 +151,30 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
|
||||
data.run_type = BenchmarkReporter::Run::RT_Aggregate;
|
||||
data.aggregate_name = Stat.name_;
|
||||
data.report_label = report_label;
|
||||
data.iterations = run_iterations;
|
||||
|
||||
// It is incorrect to say that an aggregate is computed over
|
||||
// run's iterations, because those iterations already got averaged.
|
||||
// Similarly, if there are N repetitions with 1 iterations each,
|
||||
// an aggregate will be computed over N measurements, not 1.
|
||||
// Thus it is best to simply use the count of separate reports.
|
||||
data.iterations = reports.size();
|
||||
|
||||
data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);
|
||||
data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);
|
||||
|
||||
// We will divide these times by data.iterations when reporting, but the
|
||||
// data.iterations is not nessesairly the scale of these measurements,
|
||||
// because in each repetition, these timers are sum over all the iterations.
|
||||
// And if we want to say that the stats are over N repetitions and not
|
||||
// M iterations, we need to multiply these by (N/M).
|
||||
data.real_accumulated_time *= iteration_rescale_factor;
|
||||
data.cpu_accumulated_time *= iteration_rescale_factor;
|
||||
|
||||
data.time_unit = reports[0].time_unit;
|
||||
|
||||
// user counters
|
||||
for (auto const& kv : counter_stats) {
|
||||
// Do *NOT* rescale the custom counters. They are already properly scaled.
|
||||
const auto uc_stat = Stat.compute_(kv.second.s);
|
||||
auto c = Counter(uc_stat, counter_stats[kv.first].c.flags,
|
||||
counter_stats[kv.first].c.oneK);
|
||||
|
||||
@ -44,9 +44,11 @@ SubMap& GetSubstitutions() {
|
||||
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
|
||||
{"%int", "[ ]*[0-9]+"},
|
||||
{" %s ", "[ ]+"},
|
||||
{"%time", "[ ]*[0-9]{1,6} ns"},
|
||||
{"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"},
|
||||
{"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
|
||||
{"%time", "[ ]*[0-9]+ ns"},
|
||||
{"%console_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns [ ]*[0-9]+"},
|
||||
{"%console_time_only_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns"},
|
||||
{"%console_us_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us [ ]*[0-9]+"},
|
||||
{"%console_us_time_only_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us"},
|
||||
{"%csv_header",
|
||||
"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
|
||||
"items_per_second,label,error_occurred,error_message"},
|
||||
|
||||
@ -239,11 +239,12 @@ void BM_Repeat(benchmark::State& state) {
|
||||
}
|
||||
// need two repetitions min to be able to output any aggregate output
|
||||
BENCHMARK(BM_Repeat)->Repetitions(2);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
|
||||
{"^BM_Repeat/repeats:2 %console_report$"},
|
||||
{"^BM_Repeat/repeats:2_mean %console_report$"},
|
||||
{"^BM_Repeat/repeats:2_median %console_report$"},
|
||||
{"^BM_Repeat/repeats:2_stddev %console_report$"}});
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{"^BM_Repeat/repeats:2 %console_report$"},
|
||||
{"^BM_Repeat/repeats:2 %console_report$"},
|
||||
{"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
|
||||
{"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
|
||||
{"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
@ -254,14 +255,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:2_median\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next}});
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:2\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
|
||||
@ -269,12 +273,13 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
|
||||
// but for two repetitions, mean and median is the same, so let's repeat..
|
||||
BENCHMARK(BM_Repeat)->Repetitions(3);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_mean %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_median %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_stddev %console_report$"}});
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3 %console_report$"},
|
||||
{"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
|
||||
{"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
|
||||
{"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
@ -288,14 +293,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_median\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next}});
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
@ -304,13 +312,14 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
|
||||
// median differs between even/odd number of repetitions, so just to be sure
|
||||
BENCHMARK(BM_Repeat)->Repetitions(4);
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4_mean %console_report$"},
|
||||
{"^BM_Repeat/repeats:4_median %console_report$"},
|
||||
{"^BM_Repeat/repeats:4_stddev %console_report$"}});
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4 %console_report$"},
|
||||
{"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
|
||||
{"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
|
||||
{"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
@ -327,14 +336,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 4,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:4_median\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 4,$", MR_Next},
|
||||
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
|
||||
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next}});
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 4,$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:4\",%csv_report$"},
|
||||
{"^\"BM_Repeat/repeats:4\",%csv_report$"},
|
||||
@ -362,25 +374,29 @@ void BM_SummaryRepeat(benchmark::State& state) {
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_median %console_report$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
|
||||
ADD_CASES(
|
||||
TC_ConsoleOut,
|
||||
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
|
||||
{"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
|
||||
ADD_CASES(TC_JSONOut,
|
||||
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
|
||||
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
|
||||
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
|
||||
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next}});
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
|
||||
{"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
|
||||
@ -394,25 +410,29 @@ void BM_SummaryDisplay(benchmark::State& state) {
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
|
||||
{"^BM_SummaryDisplay/repeats:2_mean %console_report$"},
|
||||
{"^BM_SummaryDisplay/repeats:2_median %console_report$"},
|
||||
{"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}});
|
||||
ADD_CASES(
|
||||
TC_ConsoleOut,
|
||||
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
|
||||
{"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
|
||||
{"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
|
||||
{"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
|
||||
ADD_CASES(TC_JSONOut,
|
||||
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
|
||||
{"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
|
||||
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
|
||||
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
|
||||
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next}});
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut,
|
||||
{{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
|
||||
{"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
|
||||
@ -428,27 +448,33 @@ BENCHMARK(BM_RepeatTimeUnit)
|
||||
->Repetitions(3)
|
||||
->ReportAggregatesOnly()
|
||||
->Unit(benchmark::kMicrosecond);
|
||||
ADD_CASES(TC_ConsoleOut,
|
||||
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
|
||||
ADD_CASES(
|
||||
TC_ConsoleOut,
|
||||
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
|
||||
"]*3$"},
|
||||
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
|
||||
"]*3$"}});
|
||||
ADD_CASES(TC_JSONOut,
|
||||
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
|
||||
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
|
||||
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"time_unit\": \"us\",?$"},
|
||||
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
|
||||
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"time_unit\": \"us\",?$"},
|
||||
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
|
||||
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"time_unit\": \"us\",?$"}});
|
||||
ADD_CASES(TC_CSVOut,
|
||||
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
|
||||
@ -465,55 +491,92 @@ const auto UserStatistics = [](const std::vector<double>& v) {
|
||||
};
|
||||
void BM_UserStats(benchmark::State& state) {
|
||||
for (auto _ : state) {
|
||||
state.SetIterationTime(150 / 10e8);
|
||||
}
|
||||
}
|
||||
// clang-format off
|
||||
BENCHMARK(BM_UserStats)
|
||||
->Repetitions(3)
|
||||
->Iterations(5)
|
||||
->UseManualTime()
|
||||
->ComputeStatistics("", UserStatistics);
|
||||
// clang-format on
|
||||
|
||||
// check that user-provided stats is calculated, and is after the default-ones
|
||||
// empty string as name is intentional, it would sort before anything else
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
|
||||
{"^BM_UserStats/repeats:3 %console_report$"},
|
||||
{"^BM_UserStats/repeats:3 %console_report$"},
|
||||
{"^BM_UserStats/repeats:3_mean %console_report$"},
|
||||
{"^BM_UserStats/repeats:3_median %console_report$"},
|
||||
{"^BM_UserStats/repeats:3_stddev %console_report$"},
|
||||
{"^BM_UserStats/repeats:3_ %console_report$"}});
|
||||
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3_median\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/repeats:3_\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"\",$", MR_Next}});
|
||||
ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
|
||||
{"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
|
||||
ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
|
||||
"]* 150 ns %time [ ]*5$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
|
||||
"]* 150 ns %time [ ]*5$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
|
||||
"]* 150 ns %time [ ]*5$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/"
|
||||
"manual_time_mean [ ]* 150 ns %time [ ]*3$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/"
|
||||
"manual_time_median [ ]* 150 ns %time [ ]*3$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/"
|
||||
"manual_time_stddev [ ]* 0 ns %time [ ]*3$"},
|
||||
{"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
|
||||
"[ ]* 150 ns %time [ ]*3$"}});
|
||||
ADD_CASES(
|
||||
TC_JSONOut,
|
||||
{{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"iterations\": 5,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"iterations\": 5,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"iteration\",$", MR_Next},
|
||||
{"\"iterations\": 5,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"real_time\": %float,$", MR_Next},
|
||||
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
|
||||
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
|
||||
MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"\",$", MR_Next},
|
||||
{"\"iterations\": 3,$", MR_Next},
|
||||
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
|
||||
ADD_CASES(
|
||||
TC_CSVOut,
|
||||
{{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/"
|
||||
"manual_time_median\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/"
|
||||
"manual_time_stddev\",%csv_report$"},
|
||||
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
|
||||
|
||||
// ========================================================================= //
|
||||
// --------------------------- TEST CASES END ------------------------------ //
|
||||
|
||||
@ -45,9 +45,9 @@ ADD_CASES(
|
||||
"t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
|
||||
"t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
|
||||
"t4_1048576Base1024=1024k$"},
|
||||
{"^BM_Counters_Thousands/repeats:2_stddev %console_report "
|
||||
"t0_1000000DefaultBase=0 t1_1000000Base1000=0 t2_1000000Base1024=0 "
|
||||
"t3_1048576Base1000=0 t4_1048576Base1024=0$"},
|
||||
{"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ "
|
||||
"]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 "
|
||||
"t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"},
|
||||
});
|
||||
ADD_CASES(TC_JSONOut,
|
||||
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
|
||||
@ -82,7 +82,7 @@ ADD_CASES(TC_JSONOut,
|
||||
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"mean\",$", MR_Next},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"real_time\": %float,$", MR_Next},
|
||||
{"\"cpu_time\": %float,$", MR_Next},
|
||||
{"\"time_unit\": \"ns\",$", MR_Next},
|
||||
@ -97,7 +97,7 @@ ADD_CASES(TC_JSONOut,
|
||||
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"median\",$", MR_Next},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"real_time\": %float,$", MR_Next},
|
||||
{"\"cpu_time\": %float,$", MR_Next},
|
||||
{"\"time_unit\": \"ns\",$", MR_Next},
|
||||
@ -112,7 +112,7 @@ ADD_CASES(TC_JSONOut,
|
||||
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
|
||||
{"\"run_type\": \"aggregate\",$", MR_Next},
|
||||
{"\"aggregate_name\": \"stddev\",$", MR_Next},
|
||||
{"\"iterations\": %int,$", MR_Next},
|
||||
{"\"iterations\": 2,$", MR_Next},
|
||||
{"\"real_time\": %float,$", MR_Next},
|
||||
{"\"cpu_time\": %float,$", MR_Next},
|
||||
{"\"time_unit\": \"ns\",$", MR_Next},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user