mirror of
https://github.com/google/benchmark.git
synced 2024-12-28 05:20:14 +08:00
4c2e32f1d0
It may be useful for those wishing to further post-process JSON results, but it is mainly geared towards better support for run interleaving, where results from the same family may not be close-by in the JSON. While we won't be able to do much about that for outputs, the tools can and perhaps should reorder the results to that at least in their output they are in proper order, not run order. Note that this only counts the families that were filtered-in, so if e.g. there were three families, and we filtered-out the second one, the two families (which were first and third) will have family indexes 0 and 1.
291 lines
14 KiB
C++
291 lines
14 KiB
C++
|
|
#undef NDEBUG
|
|
|
|
#include "benchmark/benchmark.h"
|
|
#include "output_test.h"
|
|
|
|
// @todo: <jpmag> this checks the full output at once; the rule for
|
|
// CounterSet1 was failing because it was not matching "^[-]+$".
|
|
// @todo: <jpmag> check that the counters are vertically aligned.
|
|
ADD_CASES(
|
|
TC_ConsoleOut,
|
|
{
|
|
// keeping these lines long improves readability, so:
|
|
// clang-format off
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
|
|
{"^[-]+$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
|
|
{"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
|
|
// clang-format on
|
|
});
|
|
ADD_CASES(TC_CSVOut, {{"%csv_header,"
|
|
"\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_Counters_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreads}},
|
|
{"Bar", {2, bm::Counter::kAvgThreads}},
|
|
{"Baz", {4, bm::Counter::kAvgThreads}},
|
|
{"Bat", {8, bm::Counter::kAvgThreads}},
|
|
{"Frob", {16, bm::Counter::kAvgThreads}},
|
|
{"Lob", {32, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 0,$", MR_Next},
|
|
{"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabular(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
|
|
CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
|
|
CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
|
|
|
|
// ========================================================================= //
|
|
// -------------------- Tabular+Rate Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
void BM_CounterRates_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
// This test requires a non-zero CPU time to avoid divide-by-zero
|
|
benchmark::DoNotOptimize(state.iterations());
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {1, bm::Counter::kAvgThreadsRate}},
|
|
{"Bar", {2, bm::Counter::kAvgThreadsRate}},
|
|
{"Baz", {4, bm::Counter::kAvgThreadsRate}},
|
|
{"Bat", {8, bm::Counter::kAvgThreadsRate}},
|
|
{"Frob", {16, bm::Counter::kAvgThreadsRate}},
|
|
{"Lob", {32, bm::Counter::kAvgThreadsRate}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 1,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
|
|
MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float,$", MR_Next},
|
|
{"\"Frob\": %float,$", MR_Next},
|
|
{"\"Lob\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
|
|
"%float,%float,%float,%float,%float,%float$"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckTabularRate(Results const& e) {
|
|
double t = e.DurationCPUTime();
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001);
|
|
CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
|
|
&CheckTabularRate);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters
|
|
void BM_CounterSet0_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bar", {20, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 2,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet0(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
|
|
|
|
// again.
|
|
void BM_CounterSet1_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {15, bm::Counter::kAvgThreads}},
|
|
{"Bar", {25, bm::Counter::kAvgThreads}},
|
|
{"Baz", {45, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 3,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bar\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
|
|
"%float,,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet1(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
|
|
CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Tabular Counters Output ----------------------- //
|
|
// ========================================================================= //
|
|
|
|
// set only some of the counters, different set now.
|
|
void BM_CounterSet2_Tabular(benchmark::State& state) {
|
|
for (auto _ : state) {
|
|
}
|
|
namespace bm = benchmark;
|
|
state.counters.insert({
|
|
{"Foo", {10, bm::Counter::kAvgThreads}},
|
|
{"Bat", {30, bm::Counter::kAvgThreads}},
|
|
{"Baz", {40, bm::Counter::kAvgThreads}},
|
|
});
|
|
}
|
|
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
|
|
ADD_CASES(TC_JSONOut,
|
|
{{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
|
|
{"\"family_index\": 4,$", MR_Next},
|
|
{"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
|
|
{"\"run_type\": \"iteration\",$", MR_Next},
|
|
{"\"repetitions\": 1,$", MR_Next},
|
|
{"\"repetition_index\": 0,$", MR_Next},
|
|
{"\"threads\": 1,$", MR_Next},
|
|
{"\"iterations\": %int,$", MR_Next},
|
|
{"\"real_time\": %float,$", MR_Next},
|
|
{"\"cpu_time\": %float,$", MR_Next},
|
|
{"\"time_unit\": \"ns\",$", MR_Next},
|
|
{"\"Bat\": %float,$", MR_Next},
|
|
{"\"Baz\": %float,$", MR_Next},
|
|
{"\"Foo\": %float$", MR_Next},
|
|
{"}", MR_Next}});
|
|
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
|
|
",%float,%float,%float,,"}});
|
|
// VS2013 does not allow this function to be passed as a lambda argument
|
|
// to CHECK_BENCHMARK_RESULTS()
|
|
void CheckSet2(Results const& e) {
|
|
CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
|
|
CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
|
|
CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
|
|
}
|
|
CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
|
|
|
|
// ========================================================================= //
|
|
// --------------------------- TEST CASES END ------------------------------ //
|
|
// ========================================================================= //
|
|
|
|
int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
|