1
0
mirror of https://github.com/google/benchmark.git synced 2025-01-16 06:40:13 +08:00

Unit testing: add facilities to check benchmark results.

This is needed for examining the values of user counters (needed
for ). It is also needed for checking the values of standard
benchmark results like items_processed or complexities (for example,
checking the standard deviation is needed for unit testing 
as discussed in ).
This commit is contained in:
Joao Paulo Magalhaes 2017-04-28 15:02:27 +01:00
parent 693a43013d
commit 6452883027
3 changed files with 232 additions and 8 deletions

View File

@ -58,6 +58,79 @@ int SetSubstitutions(
// Run all output tests. // Run all output tests.
void RunOutputTests(int argc, char* argv[]); void RunOutputTests(int argc, char* argv[]);
// ========================================================================= //
// ------------------------- Results checking ------------------------------ //
// ========================================================================= //
struct ResultsCheckerEntry;
typedef std::function< void(ResultsCheckerEntry const&) > ResultsCheckFn;
// Class to test the results of a benchmark.
// It inspects the results by looking at the CSV output of a subscribed
// benchmark.
struct ResultsCheckerEntry {
std::string name;
std::map< std::string, std::string > values;
ResultsCheckFn check_fn;
// int NumThreads() const; // TODO
// double duration_real_time() const {} // TODO
// double duration_cpu_time() const {} // TODO
// get the string for a result by name, or nullptr if the name
// is not found
const std::string* Get(std::string const& entry_name) const {
auto it = values.find(entry_name);
if(it == values.end()) return nullptr;
return &it->second;
}
// get a result by name, parsed as a specific type.
// For counters, use GetCounterAs instead.
template< class T > T GetAs(std::string const& entry_name) const {
auto *sv = Get(entry_name);
CHECK(sv != nullptr && !sv->empty());
std::stringstream ss;
ss << *sv;
T out;
ss >> out;
CHECK(!ss.fail());
return out;
}
// counters are written as doubles, so they have to be read first
// as a double, and only then converted to the asked type.
template< class T > T GetCounterAs(std::string const& entry_name) const {
double dval = GetAs< double >(entry_name);
T tval = static_cast< T >(dval);
return tval;
}
};
#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value)) \
<< "\n" << __FILE__ << ":" << __LINE__ << ": " \
<< entry.name << ": expected (" << #var_type << ")" \
<< var_name << "=" << entry.GetAs< var_type >(var_name) \
<< " to be " #relationship " to " << (value);
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
_CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
_CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
#define CHECK_BENCHMARK_RESULTS(bm_name, checker_function) \
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function)
// Add a function to check the (CSV) results of a benchmark. These
// functions will be called only after the output was successfully
// checked.
size_t AddChecker(const char* bm_name, ResultsCheckFn fn);
// ========================================================================= // // ========================================================================= //
// --------------------------- Misc Utilities ------------------------------ // // --------------------------- Misc Utilities ------------------------------ //
// ========================================================================= // // ========================================================================= //

View File

@ -34,6 +34,8 @@ SubMap& GetSubstitutions() {
static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
static SubMap map = { static SubMap map = {
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
// human-readable float
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
{"%int", "[ ]*[0-9]+"}, {"%int", "[ ]*[0-9]+"},
{" %s ", "[ ]+"}, {" %s ", "[ ]+"},
{"%time", "[ ]*[0-9]{1,5} ns"}, {"%time", "[ ]*[0-9]{1,5} ns"},
@ -146,8 +148,139 @@ class TestReporter : public benchmark::BenchmarkReporter {
std::vector<benchmark::BenchmarkReporter *> reporters_; std::vector<benchmark::BenchmarkReporter *> reporters_;
}; };
} }
} // end namespace internal } // end namespace internal
// ========================================================================= //
// -------------------------- Results checking ----------------------------- //
// ========================================================================= //
namespace internal {
// Utility class to manage subscribers for checking benchmark results.
// It works by parsing the CSV output to read the results.
class ResultsChecker {
public:
std::map< std::string, ResultsCheckerEntry > results;
std::vector< std::string > result_names;
void Add(const std::string& entry_name, ResultsCheckFn fn);
void CheckResults(std::stringstream& output);
private:
ResultsCheckerEntry* Find_(const std::string& entry_name);
void SetHeader_(const std::string& csv_header);
void SetValues_(const std::string& entry_csv_line);
std::vector< std::string > SplitCsv_(std::string const& line);
};
// store the static ResultsChecker in a function to prevent initialization
// order problems
ResultsChecker& GetResultsChecker() {
static ResultsChecker rc;
return rc;
}
// add a results checker for a benchmark
void ResultsChecker::Add(const std::string& entry_name, ResultsCheckFn fn) {
results[entry_name] = {entry_name, {}, fn};
}
// check the results of all subscribed benchmarks
void ResultsChecker::CheckResults(std::stringstream& output)
{
// first reset the stream to the start
{
auto start = std::ios::streampos(0);
// clear before calling tellg()
output.clear();
// seek to zero only when needed
if(output.tellg() > start) output.seekg(start);
// and just in case
output.clear();
}
// now go over every line and publish it to the ResultsChecker
std::string line;
bool on_first = true;
while (output.eof() == false) {
CHECK(output.good());
std::getline(output, line);
if (on_first) {
SetHeader_(line); // this is important
on_first = false;
continue;
}
SetValues_(line);
}
// finally we can call the subscribed check functions
for(const auto& p : results) {
CHECK(p.second.check_fn);
p.second.check_fn(p.second);
}
}
// prepare for the names in this header
void ResultsChecker::SetHeader_(const std::string& csv_header) {
result_names = SplitCsv_(csv_header);
}
// set the values for subscribed benchmarks, and silently ignore all others
void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
CHECK(!result_names.empty());
auto vals = SplitCsv_(entry_csv_line);
if(vals.empty()) return;
CHECK_EQ(vals.size(), result_names.size());
ResultsCheckerEntry* entry = Find_(vals[0]);
if(!entry) return;
for (size_t i = 1, e = vals.size(); i < e; ++i) {
entry->values[result_names[i]] = vals[i];
}
}
// find a subscribed benchmark, or return null
ResultsCheckerEntry* ResultsChecker::Find_(const std::string& entry_name) {
auto it = results.find(entry_name);
if(it == results.end()) return nullptr;
return &it->second;
}
// a quick'n'dirty csv splitter (eliminating quotes)
std::vector< std::string > ResultsChecker::SplitCsv_(std::string const& line) {
std::vector< std::string > out;
if(line.empty()) return out;
if(!result_names.empty()) out.reserve(result_names.size());
size_t prev = 0, pos = line.find_first_of(','), curr = pos;
while(pos != line.npos) {
CHECK(curr > 0);
if(line[prev] == '"') ++prev;
if(line[curr-1] == '"') --curr;
out.push_back(line.substr(prev, curr-prev));
prev = pos + 1;
pos = line.find_first_of(',', pos + 1);
curr = pos;
}
curr = line.size();
if(line[prev] == '"') ++prev;
if(line[curr-1] == '"') --curr;
out.push_back(line.substr(prev, curr-prev));
return out;
}
} // end namespace internal
size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
{
auto &rc = internal::GetResultsChecker();
rc.Add(bm_name, fn);
return rc.results.size();
}
// ========================================================================= // // ========================================================================= //
// -------------------------- Public API Definitions------------------------ // // -------------------------- Public API Definitions------------------------ //
// ========================================================================= // // ========================================================================= //
@ -237,4 +370,10 @@ void RunOutputTests(int argc, char* argv[]) {
std::cout << "\n"; std::cout << "\n";
} }
// now that we know the output is as expected, we can dispatch
// the checks to subscribees.
auto &csv = TestCases[2];
CHECK(strcmp(csv.name, "CSVReporter") == 0); // would use == but gcc spits a warning
internal::GetResultsChecker().CheckResults(csv.out_stream);
} }

View File

@ -25,7 +25,7 @@ void BM_Counters_Simple(benchmark::State& state) {
state.counters["bar"] = 2; state.counters["bar"] = 2;
} }
BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32); BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%float foo=%float$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
@ -35,23 +35,28 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"foo\": %float$", MR_Next}, {"\"foo\": %float$", MR_Next},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) {
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
});
// ========================================================================= // // ========================================================================= //
// --------------------- Counters+Items+Bytes/s Output --------------------- // // --------------------- Counters+Items+Bytes/s Output --------------------- //
// ========================================================================= // // ========================================================================= //
namespace { int num_calls1 = 0; }
void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
state.counters["foo"] = 1; state.counters["foo"] = 1;
state.counters["bar"] = 2; state.counters["bar"] = ++num_calls1;
state.SetItemsProcessed(150);
state.SetBytesProcessed(364); state.SetBytesProcessed(364);
state.SetItemsProcessed(150);
} }
BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32); BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32);
ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_WithBytesAndItemsPSec %console_report " {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
"bar=%float foo=%float +%floatB/s +%float items/s$"}}); "bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
@ -64,6 +69,13 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"}", MR_Next}}); {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
"%csv_bytes_items_report,%float,%float$"}}); "%csv_bytes_items_report,%float,%float$"}});
CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
[](ResultsCheckerEntry const& e) {
CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
//TODO CHECK_RESULT_VALUE(e, int, "bytes_per_second", EQ, 364 / e.duration_cpu_time());
//TODO CHECK_RESULT_VALUE(e, int, "items_per_second", EQ, 150 / e.duration_cpu_time());
});
// ========================================================================= // // ========================================================================= //
// ------------------------- Rate Counters Output -------------------------- // // ------------------------- Rate Counters Output -------------------------- //
@ -77,7 +89,7 @@ void BM_Counters_Rate(benchmark::State& state) {
state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
} }
BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32); BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%float foo=%float$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
@ -99,7 +111,7 @@ void BM_Counters_Threads(benchmark::State& state) {
state.counters["bar"] = 2; state.counters["bar"] = 2;
} }
BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%float foo=%float$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
@ -122,7 +134,7 @@ void BM_Counters_AvgThreads(benchmark::State& state) {
state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
} }
BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%float foo=%float$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},
@ -145,7 +157,7 @@ void BM_Counters_AvgThreadsRate(benchmark::State& state) {
state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
} }
BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%float foo=%float$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
{"\"iterations\": %int,$", MR_Next}, {"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},