Unit testing: add facilities to check benchmark results.

This is needed for examining the values of user counters (needed
for #348). It is also needed for checking the values of standard
benchmark results like items_processed or complexities (for example,
checking the standard deviation is needed for unit testing #357
as discussed in #362).

test/output_test.h

@@ -58,6 +58,79 @@ int SetSubstitutions(
 // Run all output tests.
 void RunOutputTests(int argc, char* argv[]);
 
+// ========================================================================= //
+// ------------------------- Results checking ------------------------------ //
+// ========================================================================= //
+
+struct ResultsCheckerEntry;
+typedef std::function< void(ResultsCheckerEntry const&) > ResultsCheckFn;
+
+// Class to test the results of a benchmark.
+// It inspects the results by looking at the CSV output of a subscribed
+// benchmark.
+struct ResultsCheckerEntry {
+  std::string name;
+  std::map< std::string, std::string > values;
+  ResultsCheckFn check_fn;
+
+  // int NumThreads() const; // TODO
+  // double duration_real_time() const {} // TODO
+  // double duration_cpu_time() const {} // TODO
+
+  // get the string for a result by name, or nullptr if the name
+  // is not found
+  const std::string* Get(std::string const& entry_name) const {
+    auto it = values.find(entry_name);
+    if(it == values.end()) return nullptr;
+    return &it->second;
+  }
+
+  // get a result by name, parsed as a specific type.
+  // For counters, use GetCounterAs instead.
+  template< class T > T GetAs(std::string const& entry_name) const {
+    auto *sv = Get(entry_name);
+    CHECK(sv != nullptr && !sv->empty());
+    std::stringstream ss;
+    ss << *sv;
+    T out;
+    ss >> out;
+    CHECK(!ss.fail());
+    return out;
+  }
+
+  // counters are written as doubles, so they have to be read first
+  // as a double, and only then converted to the asked type.
+  template< class T > T GetCounterAs(std::string const& entry_name) const {
+    double dval = GetAs< double >(entry_name);
+    T tval = static_cast< T >(dval);
+    return tval;
+  }
+
+
+};
+
+
+#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \
+    CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value)) \
+                                << "\n" << __FILE__ << ":" << __LINE__ << ": "  \
+                                << entry.name << ": expected (" << #var_type << ")" \
+                                << var_name << "=" << entry.GetAs< var_type >(var_name) \
+                                << " to be " #relationship " to " << (value);
+
+#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
+    _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value)
+
+#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
+    _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value)
+
+#define CHECK_BENCHMARK_RESULTS(bm_name, checker_function)              \
+    size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function)
+
+// Add a function to check the (CSV) results of a benchmark. These
+// functions will be called only after the output was successfully
+// checked.
+size_t AddChecker(const char* bm_name, ResultsCheckFn fn);
+
 // ========================================================================= //
 // --------------------------- Misc Utilities ------------------------------ //
 // ========================================================================= //

test/output_test_helper.cc

@@ -34,6 +34,8 @@ SubMap& GetSubstitutions() {
   static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
   static SubMap map = {
       {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
+      // human-readable float
+      {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
       {"%int", "[ ]*[0-9]+"},
       {" %s ", "[ ]+"},
       {"%time", "[ ]*[0-9]{1,5} ns"},
@@ -146,8 +148,139 @@ class TestReporter : public benchmark::BenchmarkReporter {
   std::vector<benchmark::BenchmarkReporter *> reporters_;
 };
 }
+
 }  // end namespace internal
 
+// ========================================================================= //
+// -------------------------- Results checking ----------------------------- //
+// ========================================================================= //
+
+namespace internal {
+
+// Utility class to manage subscribers for checking benchmark results.
+// It works by parsing the CSV output to read the results.
+class ResultsChecker {
+ public:
+
+  std::map< std::string, ResultsCheckerEntry > results;
+  std::vector< std::string > result_names;
+
+  void Add(const std::string& entry_name, ResultsCheckFn fn);
+
+  void CheckResults(std::stringstream& output);
+
+ private:
+
+  ResultsCheckerEntry* Find_(const std::string& entry_name);
+
+  void SetHeader_(const std::string& csv_header);
+  void SetValues_(const std::string& entry_csv_line);
+
+  std::vector< std::string > SplitCsv_(std::string const& line);
+
+};
+
+// store the static ResultsChecker in a function to prevent initialization
+// order problems
+ResultsChecker& GetResultsChecker() {
+  static ResultsChecker rc;
+  return rc;
+}
+
+// add a results checker for a benchmark
+void ResultsChecker::Add(const std::string& entry_name, ResultsCheckFn fn) {
+  results[entry_name] = {entry_name, {}, fn};
+}
+
+// check the results of all subscribed benchmarks
+void ResultsChecker::CheckResults(std::stringstream& output)
+{
+  // first reset the stream to the start
+  {
+    auto start = std::ios::streampos(0);
+    // clear before calling tellg()
+    output.clear();
+    // seek to zero only when needed
+    if(output.tellg() > start) output.seekg(start);
+    // and just in case
+    output.clear();
+  }
+  // now go over every line and publish it to the ResultsChecker
+  std::string line;
+  bool on_first = true;
+  while (output.eof() == false) {
+    CHECK(output.good());
+    std::getline(output, line);
+    if (on_first) {
+      SetHeader_(line); // this is important
+      on_first = false;
+      continue;
+    }
+    SetValues_(line);
+  }
+  // finally we can call the subscribed check functions
+  for(const auto& p : results) {
+    CHECK(p.second.check_fn);
+    p.second.check_fn(p.second);
+  }
+}
+
+// prepare for the names in this header
+void ResultsChecker::SetHeader_(const std::string& csv_header) {
+  result_names = SplitCsv_(csv_header);
+}
+
+// set the values for subscribed benchmarks, and silently ignore all others
+void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
+  CHECK(!result_names.empty());
+  auto vals = SplitCsv_(entry_csv_line);
+  if(vals.empty()) return;
+  CHECK_EQ(vals.size(), result_names.size());
+  ResultsCheckerEntry* entry = Find_(vals[0]);
+  if(!entry) return;
+  for (size_t i = 1, e = vals.size(); i < e; ++i) {
+    entry->values[result_names[i]] = vals[i];
+  }
+}
+
+// find a subscribed benchmark, or return null
+ResultsCheckerEntry* ResultsChecker::Find_(const std::string& entry_name) {
+  auto it = results.find(entry_name);
+  if(it == results.end()) return nullptr;
+  return &it->second;
+}
+
+// a quick'n'dirty csv splitter (eliminating quotes)
+std::vector< std::string > ResultsChecker::SplitCsv_(std::string const& line) {
+  std::vector< std::string > out;
+  if(line.empty()) return out;
+  if(!result_names.empty()) out.reserve(result_names.size());
+  size_t prev = 0, pos = line.find_first_of(','), curr = pos;
+  while(pos != line.npos) {
+    CHECK(curr > 0);
+    if(line[prev] == '"') ++prev;
+    if(line[curr-1] == '"') --curr;
+    out.push_back(line.substr(prev, curr-prev));
+    prev = pos + 1;
+    pos = line.find_first_of(',', pos + 1);
+    curr = pos;
+  }
+  curr = line.size();
+  if(line[prev] == '"') ++prev;
+  if(line[curr-1] == '"') --curr;
+  out.push_back(line.substr(prev, curr-prev));
+  return out;
+}
+
+}  // end namespace internal
+
+size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
+{
+  auto &rc = internal::GetResultsChecker();
+  rc.Add(bm_name, fn);
+  return rc.results.size();
+}
+
 // ========================================================================= //
 // -------------------------- Public API Definitions------------------------ //
 // ========================================================================= //
@@ -237,4 +370,10 @@ void RunOutputTests(int argc, char* argv[]) {
 
     std::cout << "\n";
   }
+
+  // now that we know the output is as expected, we can dispatch
+  // the checks to subscribees.
+  auto &csv = TestCases[2];
+  CHECK(strcmp(csv.name, "CSVReporter") == 0); // would use == but gcc spits a warning
+  internal::GetResultsChecker().CheckResults(csv.out_stream);
 }

test/user_counters_test.cc

@@ -25,7 +25,7 @@ void BM_Counters_Simple(benchmark::State& state) {
   state.counters["bar"] = 2;
 }
 BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%float foo=%float$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -35,23 +35,28 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
                        {"\"foo\": %float$", MR_Next},
                        {"}", MR_Next}});
 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
+CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) {
+  CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
+  CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
+});
 
 // ========================================================================= //
 // --------------------- Counters+Items+Bytes/s Output --------------------- //
 // ========================================================================= //
 
+namespace { int num_calls1 = 0; }
 void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
   while (state.KeepRunning()) {
   }
   state.counters["foo"] = 1;
-  state.counters["bar"] = 2;
+  state.counters["bar"] = ++num_calls1;
-  state.SetItemsProcessed(150);
   state.SetBytesProcessed(364);
+  state.SetItemsProcessed(150);
 }
 BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32);
 ADD_CASES(TC_ConsoleOut,
           {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
-            "bar=%float foo=%float +%floatB/s +%float items/s$"}});
+            "bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -64,6 +69,13 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
                        {"}", MR_Next}});
 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
                        "%csv_bytes_items_report,%float,%float$"}});
+CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
+                        [](ResultsCheckerEntry const& e) {
+  CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
+  CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
+  //TODO CHECK_RESULT_VALUE(e, int, "bytes_per_second", EQ, 364 / e.duration_cpu_time());
+  //TODO CHECK_RESULT_VALUE(e, int, "items_per_second", EQ, 150 / e.duration_cpu_time());
+});
 
 // ========================================================================= //
 // ------------------------- Rate Counters Output -------------------------- //
@@ -77,7 +89,7 @@ void BM_Counters_Rate(benchmark::State& state) {
   state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
 }
 BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%float foo=%float$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -99,7 +111,7 @@ void BM_Counters_Threads(benchmark::State& state) {
   state.counters["bar"] = 2;
 }
 BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%float foo=%float$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -122,7 +134,7 @@ void BM_Counters_AvgThreads(benchmark::State& state) {
   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
 }
 BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%float foo=%float$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},
@@ -145,7 +157,7 @@ void BM_Counters_AvgThreadsRate(benchmark::State& state) {
   state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
 }
 BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%float foo=%float$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}});
 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
                        {"\"iterations\": %int,$", MR_Next},
                        {"\"real_time\": %int,$", MR_Next},