github-repos/benchmark
1
0
Fork 0
mirror of https://github.com/google/benchmark.git synced 2024-12-27 13:00:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Track 'type' of the run - is it an actual measurement, or an aggregate. (#658)

Browse Source 
This is *only* exposed in the JSON. Not in CSV, which is deprecated.

This *only* supposed to track these two states.
An additional field could later track which aggregate this is,
specifically (statistic name, rms, bigo, ...)

The motivation is that we already have ReportAggregatesOnly,
but it affects the entire reports, both the display,
and the reporters (json files), which isn't ideal.

It would be very useful to have a 'display aggregates only' option,
both in the library's console reporter, and the python tooling,
This will be especially needed for the 'store separate iterations'.
This commit is contained in:
Roman Lebedev 2018-08-28 18:11:36 +03:00 committed by GitHub
parent 9a179cb93f
commit 8688c5c4cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 84 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/benchmark/benchmark.h
View File

@ -1276,8 +1276,11 @@ class BenchmarkReporter {
}; };
struct Run { struct Run {
enum RunType { RT_Iteration, RT_Aggregate };
Run() Run()
: error_occurred(false), : run_type(RT_Iteration),
error_occurred(false),
iterations(1), iterations(1),
time_unit(kNanosecond), time_unit(kNanosecond),
real_accumulated_time(0), real_accumulated_time(0),
@ -1296,6 +1299,7 @@ class BenchmarkReporter {
max_bytes_used(0) {} max_bytes_used(0) {}
std::string benchmark_name; std::string benchmark_name;
RunType run_type; // is this a measurement, or an aggregate?
std::string report_label; // Empty if not set by benchmark. std::string report_label; // Empty if not set by benchmark.
bool error_occurred; bool error_occurred;
std::string error_message; std::string error_message;

2
src/complexity.cc
View File

@ -187,6 +187,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
// Get the data from the accumulator to BenchmarkReporter::Run's. // Get the data from the accumulator to BenchmarkReporter::Run's.
Run big_o; Run big_o;
big_o.run_type = BenchmarkReporter::Run::RT_Aggregate;
big_o.benchmark_name = benchmark_name + "_BigO"; big_o.benchmark_name = benchmark_name + "_BigO";
big_o.iterations = 0; big_o.iterations = 0;
big_o.real_accumulated_time = result_real.coef; big_o.real_accumulated_time = result_real.coef;
@ -204,6 +205,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
// Only add label to mean/stddev if it is same for all runs // Only add label to mean/stddev if it is same for all runs
Run rms; Run rms;
big_o.report_label = reports[0].report_label; big_o.report_label = reports[0].report_label;
rms.run_type = BenchmarkReporter::Run::RT_Aggregate;
rms.benchmark_name = benchmark_name + "_RMS"; rms.benchmark_name = benchmark_name + "_RMS";
rms.report_label = big_o.report_label; rms.report_label = big_o.report_label;
rms.iterations = 0; rms.iterations = 0;

9
src/json_reporter.cc
View File

@ -160,6 +160,15 @@ void JSONReporter::PrintRunData(Run const& run) {
std::string indent(6, ' '); std::string indent(6, ' ');
std::ostream& out = GetOutputStream(); std::ostream& out = GetOutputStream();
out << indent << FormatKV("name", run.benchmark_name) << ",\n"; out << indent << FormatKV("name", run.benchmark_name) << ",\n";
out << indent << FormatKV("run_type", [&run]() -> const char* {
switch (run.run_type) {
case BenchmarkReporter::Run::RT_Iteration:
return "iteration";
case BenchmarkReporter::Run::RT_Aggregate:
return "aggregate";
}
BENCHMARK_UNREACHABLE();
}()) << ",\n";
if (run.error_occurred) { if (run.error_occurred) {
out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n";
out << indent << FormatKV("error_message", run.error_message) << ",\n"; out << indent << FormatKV("error_message", run.error_message) << ",\n";

1
src/statistics.cc
View File

@ -150,6 +150,7 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
for (const auto& Stat : *reports[0].statistics) { for (const auto& Stat : *reports[0].statistics) {
// Get the data from the accumulator to BenchmarkReporter::Run's. // Get the data from the accumulator to BenchmarkReporter::Run's.
Run data; Run data;
data.run_type = BenchmarkReporter::Run::RT_Aggregate;
data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_; data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_;
data.report_label = report_label; data.report_label = report_label;
data.iterations = run_iterations; data.iterations = run_iterations;

2
test/complexity_test.cc
View File

@ -25,12 +25,14 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name,
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
{"^%rms_name %rms %rms[ ]*$", MR_Next}}); {"^%rms_name %rms %rms[ ]*$", MR_Next}});
AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}, {"}", MR_Next},
{"\"name\": \"%rms_name\",$"}, {"\"name\": \"%rms_name\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"rms\": %float$", MR_Next}, {"\"rms\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},

1
test/memory_manager_test.cc
View File

@ -21,6 +21,7 @@ BENCHMARK(BM_empty);
ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},

58
test/reporter_output_test.cc
View File

@ -65,6 +65,7 @@ BENCHMARK(BM_basic);
ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -86,6 +87,7 @@ BENCHMARK(BM_bytes_per_second);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}}); {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -108,6 +110,7 @@ BENCHMARK(BM_items_per_second);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}}); {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -129,6 +132,7 @@ BENCHMARK(BM_label);
ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -150,6 +154,7 @@ void BM_error(benchmark::State& state) {
BENCHMARK(BM_error); BENCHMARK(BM_error);
ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"error_occurred\": true,$", MR_Next}, {"\"error_occurred\": true,$", MR_Next},
{"\"error_message\": \"message\",$", MR_Next}}); {"\"error_message\": \"message\",$", MR_Next}});
@ -166,7 +171,8 @@ void BM_no_arg_name(benchmark::State& state) {
} }
BENCHMARK(BM_no_arg_name)->Arg(3); BENCHMARK(BM_no_arg_name)->Arg(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
@ -179,7 +185,8 @@ void BM_arg_name(benchmark::State& state) {
} }
BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
@ -193,7 +200,8 @@ void BM_arg_names(benchmark::State& state) {
BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
{"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
// ========================================================================= // // ========================================================================= //
@ -228,10 +236,15 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
{"^BM_Repeat/repeats:2_median %console_report$"}, {"^BM_Repeat/repeats:2_median %console_report$"},
{"^BM_Repeat/repeats:2_stddev %console_report$"}}); {"^BM_Repeat/repeats:2_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}}); {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
@ -246,11 +259,17 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3_median %console_report$"}, {"^BM_Repeat/repeats:3_median %console_report$"},
{"^BM_Repeat/repeats:3_stddev %console_report$"}}); {"^BM_Repeat/repeats:3_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"},
@ -267,12 +286,19 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
{"^BM_Repeat/repeats:4_median %console_report$"}, {"^BM_Repeat/repeats:4_median %console_report$"},
{"^BM_Repeat/repeats:4_stddev %console_report$"}}); {"^BM_Repeat/repeats:4_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}}); {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"},
@ -289,7 +315,8 @@ void BM_RepeatOnce(benchmark::State& state) {
} }
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
{"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
// Test that non-aggregate data is not reported // Test that non-aggregate data is not reported
@ -305,8 +332,11 @@ ADD_CASES(TC_ConsoleOut,
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
@ -327,10 +357,13 @@ ADD_CASES(TC_ConsoleOut,
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"time_unit\": \"us\",?$"}, {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"time_unit\": \"us\",?$"}}); {"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut, ADD_CASES(TC_CSVOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
@ -365,12 +398,19 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
{"^BM_UserStats/repeats:3_stddev %console_report$"}, {"^BM_UserStats/repeats:3_stddev %console_report$"},
{"^BM_UserStats/repeats:3_ %console_report$"}}); {"^BM_UserStats/repeats:3_ %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3\",$"}, {"\"name\": \"BM_UserStats/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3\",$"}, {"\"name\": \"BM_UserStats/repeats:3\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_median\",$"}, {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
{"\"name\": \"BM_UserStats/repeats:3_\",$"}}); {"\"run_type\": \"aggregate\",$", MR_Next},
{"\"name\": \"BM_UserStats/repeats:3_\",$"},
{"\"run_type\": \"aggregate\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"},
{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"},

5
test/user_counters_tabular_test.cc
View File

@ -70,6 +70,7 @@ void BM_Counters_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -114,6 +115,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -158,6 +160,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -190,6 +193,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -226,6 +230,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) {
} }
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},

10
test/user_counters_test.cc
View File

@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -70,6 +71,7 @@ ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_WithBytesAndItemsPSec %console_report " {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
"bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -110,6 +112,7 @@ ADD_CASES(
TC_ConsoleOut, TC_ConsoleOut,
{{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -142,6 +145,7 @@ BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -175,6 +179,7 @@ BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int "
"%console_report bar=%hrfloat foo=%hrfloat$"}}); "%console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -210,6 +215,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -243,6 +249,7 @@ BENCHMARK(BM_Counters_IterationInvariant);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -281,6 +288,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -317,6 +325,7 @@ BENCHMARK(BM_Counters_AvgIterations);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report "
"bar=%hrfloat foo=%hrfloat$"}}); "bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},
@ -352,6 +361,7 @@ BENCHMARK(BM_Counters_kAvgIterationsRate);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
{"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next}, {"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next},