From 6452883027b6642157a54cb5ec61c55c34f4948e Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 28 Apr 2017 15:02:27 +0100 Subject: [PATCH] Unit testing: add facilities to check benchmark results. This is needed for examining the values of user counters (needed for #348). It is also needed for checking the values of standard benchmark results like items_processed or complexities (for example, checking the standard deviation is needed for unit testing #357 as discussed in #362). --- test/output_test.h | 73 +++++++++++++++++++ test/output_test_helper.cc | 139 +++++++++++++++++++++++++++++++++++++ test/user_counters_test.cc | 28 +++++--- 3 files changed, 232 insertions(+), 8 deletions(-) diff --git a/test/output_test.h b/test/output_test.h index 57d4397a..0bc2c7ba 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -58,6 +58,79 @@ int SetSubstitutions( // Run all output tests. void RunOutputTests(int argc, char* argv[]); +// ========================================================================= // +// ------------------------- Results checking ------------------------------ // +// ========================================================================= // + +struct ResultsCheckerEntry; +typedef std::function< void(ResultsCheckerEntry const&) > ResultsCheckFn; + +// Class to test the results of a benchmark. +// It inspects the results by looking at the CSV output of a subscribed +// benchmark. +struct ResultsCheckerEntry { + std::string name; + std::map< std::string, std::string > values; + ResultsCheckFn check_fn; + + // int NumThreads() const; // TODO + // double duration_real_time() const {} // TODO + // double duration_cpu_time() const {} // TODO + + // get the string for a result by name, or nullptr if the name + // is not found + const std::string* Get(std::string const& entry_name) const { + auto it = values.find(entry_name); + if(it == values.end()) return nullptr; + return &it->second; + } + + // get a result by name, parsed as a specific type. + // For counters, use GetCounterAs instead. + template< class T > T GetAs(std::string const& entry_name) const { + auto *sv = Get(entry_name); + CHECK(sv != nullptr && !sv->empty()); + std::stringstream ss; + ss << *sv; + T out; + ss >> out; + CHECK(!ss.fail()); + return out; + } + + // counters are written as doubles, so they have to be read first + // as a double, and only then converted to the asked type. + template< class T > T GetCounterAs(std::string const& entry_name) const { + double dval = GetAs< double >(entry_name); + T tval = static_cast< T >(dval); + return tval; + } + + +}; + + +#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ + CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value)) \ + << "\n" << __FILE__ << ":" << __LINE__ << ": " \ + << entry.name << ": expected (" << #var_type << ")" \ + << var_name << "=" << entry.GetAs< var_type >(var_name) \ + << " to be " #relationship " to " << (value); + +#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) + +#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) + +#define CHECK_BENCHMARK_RESULTS(bm_name, checker_function) \ + size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function) + +// Add a function to check the (CSV) results of a benchmark. These +// functions will be called only after the output was successfully +// checked. +size_t AddChecker(const char* bm_name, ResultsCheckFn fn); + // ========================================================================= // // --------------------------- Misc Utilities ------------------------------ // // ========================================================================= // diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 0acb09bc..3209d90c 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -34,6 +34,8 @@ SubMap& GetSubstitutions() { static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, + // human-readable float + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, {"%time", "[ ]*[0-9]{1,5} ns"}, @@ -146,8 +148,139 @@ class TestReporter : public benchmark::BenchmarkReporter { std::vector reporters_; }; } + } // end namespace internal +// ========================================================================= // +// -------------------------- Results checking ----------------------------- // +// ========================================================================= // + +namespace internal { + +// Utility class to manage subscribers for checking benchmark results. +// It works by parsing the CSV output to read the results. +class ResultsChecker { + public: + + std::map< std::string, ResultsCheckerEntry > results; + std::vector< std::string > result_names; + + void Add(const std::string& entry_name, ResultsCheckFn fn); + + void CheckResults(std::stringstream& output); + + private: + + ResultsCheckerEntry* Find_(const std::string& entry_name); + + void SetHeader_(const std::string& csv_header); + void SetValues_(const std::string& entry_csv_line); + + std::vector< std::string > SplitCsv_(std::string const& line); + +}; + +// store the static ResultsChecker in a function to prevent initialization +// order problems +ResultsChecker& GetResultsChecker() { + static ResultsChecker rc; + return rc; +} + +// add a results checker for a benchmark +void ResultsChecker::Add(const std::string& entry_name, ResultsCheckFn fn) { + results[entry_name] = {entry_name, {}, fn}; +} + +// check the results of all subscribed benchmarks +void ResultsChecker::CheckResults(std::stringstream& output) +{ + // first reset the stream to the start + { + auto start = std::ios::streampos(0); + // clear before calling tellg() + output.clear(); + // seek to zero only when needed + if(output.tellg() > start) output.seekg(start); + // and just in case + output.clear(); + } + // now go over every line and publish it to the ResultsChecker + std::string line; + bool on_first = true; + while (output.eof() == false) { + CHECK(output.good()); + std::getline(output, line); + if (on_first) { + SetHeader_(line); // this is important + on_first = false; + continue; + } + SetValues_(line); + } + // finally we can call the subscribed check functions + for(const auto& p : results) { + CHECK(p.second.check_fn); + p.second.check_fn(p.second); + } +} + +// prepare for the names in this header +void ResultsChecker::SetHeader_(const std::string& csv_header) { + result_names = SplitCsv_(csv_header); +} + +// set the values for subscribed benchmarks, and silently ignore all others +void ResultsChecker::SetValues_(const std::string& entry_csv_line) { + CHECK(!result_names.empty()); + auto vals = SplitCsv_(entry_csv_line); + if(vals.empty()) return; + CHECK_EQ(vals.size(), result_names.size()); + ResultsCheckerEntry* entry = Find_(vals[0]); + if(!entry) return; + for (size_t i = 1, e = vals.size(); i < e; ++i) { + entry->values[result_names[i]] = vals[i]; + } +} + +// find a subscribed benchmark, or return null +ResultsCheckerEntry* ResultsChecker::Find_(const std::string& entry_name) { + auto it = results.find(entry_name); + if(it == results.end()) return nullptr; + return &it->second; +} + +// a quick'n'dirty csv splitter (eliminating quotes) +std::vector< std::string > ResultsChecker::SplitCsv_(std::string const& line) { + std::vector< std::string > out; + if(line.empty()) return out; + if(!result_names.empty()) out.reserve(result_names.size()); + size_t prev = 0, pos = line.find_first_of(','), curr = pos; + while(pos != line.npos) { + CHECK(curr > 0); + if(line[prev] == '"') ++prev; + if(line[curr-1] == '"') --curr; + out.push_back(line.substr(prev, curr-prev)); + prev = pos + 1; + pos = line.find_first_of(',', pos + 1); + curr = pos; + } + curr = line.size(); + if(line[prev] == '"') ++prev; + if(line[curr-1] == '"') --curr; + out.push_back(line.substr(prev, curr-prev)); + return out; +} + +} // end namespace internal + +size_t AddChecker(const char* bm_name, ResultsCheckFn fn) +{ + auto &rc = internal::GetResultsChecker(); + rc.Add(bm_name, fn); + return rc.results.size(); +} + // ========================================================================= // // -------------------------- Public API Definitions------------------------ // // ========================================================================= // @@ -237,4 +370,10 @@ void RunOutputTests(int argc, char* argv[]) { std::cout << "\n"; } + + // now that we know the output is as expected, we can dispatch + // the checks to subscribees. + auto &csv = TestCases[2]; + CHECK(strcmp(csv.name, "CSVReporter") == 0); // would use == but gcc spits a warning + internal::GetResultsChecker().CheckResults(csv.out_stream); } diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index df30d926..84736b78 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -25,7 +25,7 @@ void BM_Counters_Simple(benchmark::State& state) { state.counters["bar"] = 2; } BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%float foo=%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -35,23 +35,28 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"foo\": %float$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); +CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2); +}); // ========================================================================= // // --------------------- Counters+Items+Bytes/s Output --------------------- // // ========================================================================= // +namespace { int num_calls1 = 0; } void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { while (state.KeepRunning()) { } state.counters["foo"] = 1; - state.counters["bar"] = 2; - state.SetItemsProcessed(150); + state.counters["bar"] = ++num_calls1; state.SetBytesProcessed(364); + state.SetItemsProcessed(150); } BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " - "bar=%float foo=%float +%floatB/s +%float items/s$"}}); + "bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -64,6 +69,13 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," "%csv_bytes_items_report,%float,%float$"}}); +CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", + [](ResultsCheckerEntry const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); + //TODO CHECK_RESULT_VALUE(e, int, "bytes_per_second", EQ, 364 / e.duration_cpu_time()); + //TODO CHECK_RESULT_VALUE(e, int, "items_per_second", EQ, 150 / e.duration_cpu_time()); +}); // ========================================================================= // // ------------------------- Rate Counters Output -------------------------- // @@ -77,7 +89,7 @@ void BM_Counters_Rate(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; } BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%float foo=%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -99,7 +111,7 @@ void BM_Counters_Threads(benchmark::State& state) { state.counters["bar"] = 2; } BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%float foo=%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -122,7 +134,7 @@ void BM_Counters_AvgThreads(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; } BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%float foo=%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -145,7 +157,7 @@ void BM_Counters_AvgThreadsRate(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; } BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%float foo=%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next},