diff --git a/test/output_test.h b/test/output_test.h index 072af359..247f7005 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -148,40 +148,50 @@ struct ResultsCheckerEntry { // checked. size_t AddChecker(const char* bm_name, ResultsCheckFn fn); -#ifdef __clang__ - /* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to - * variadic macros is not portable, but works in clang, gcc, msvc, icc. - * clang requires switching off compiler warnings for pedantic mode. - * @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */ -# pragma clang diagnostic push - // warning: token pasting of ',' and __VA_ARGS__ is a GNU extension -# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" -#elif defined(__GNUC__) - /* GCC also issues a warning for zero-args calls to variadic macros. - * This warning is switched on with -pedantic and apparently there is no - * easy way to turn it off as with clang. But marking this as a system - * header works. - * @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html - * @see http://stackoverflow.com/questions/35587137/ */ -# pragma GCC system_header -#endif +//---------------------------------- +// Macros to help in result checking. Do not use them with arguments causing +// side-effects. -#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, ...) \ - CONCAT(CHECK_, relationship)(entry.getfn< var_type >(var_name), (value), ## __VA_ARGS__) \ - << "\n" << __FILE__ << ":" << __LINE__ << ": " \ - << entry.name << ": expected (" << #var_type << ")" \ - << var_name << "=" << entry.getfn< var_type >(var_name) \ - << " to be " #relationship " to " << (value); +#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ + CONCAT(CHECK_, relationship) \ + (entry.getfn< var_type >(var_name), (value)) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "expected (" << #var_type << ")" << (var_name) \ + << "=" << (entry).getfn< var_type >(var_name) \ + << " to be " #relationship " to " << (value) << "\n" -#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value, ...) \ - _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value, ## __VA_ARGS__) +// check with tolerance. eps_factor is the tolerance window, which will be +// interpreted relative to value. +#define _CHECK_RESULT_VALUE_EPS(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ + CONCAT(CHECK_, relationship) \ + (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "expected (" << #var_type << ")" << (var_name) \ + << "=" << (entry).getfn< var_type >(var_name) \ + << " to be " #relationship " to " << (value) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "with tolerance of " << (eps_factor) * (value) \ + << " (" << (eps_factor)*100. << "%), " \ + << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \ + << " (" << (((entry).getfn< var_type >(var_name) - (value)) \ + / \ + ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \ + << "%)" -#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value, ...) \ - _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value, ## __VA_ARGS__) +#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) + +#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) + +#define CHECK_RESULT_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \ + _CHECK_RESULT_VALUE_EPS(entry, GetAs, double, var_name, relationship, value, eps_factor) + +#define CHECK_COUNTER_VALUE_EPS(entry, var_name, relationship, value, eps_factor) \ + _CHECK_RESULT_VALUE_EPS(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) -#ifdef __clang__ -# pragma clang diagnostic pop -#endif #define CHECK_BENCHMARK_RESULTS(bm_name, checker_function) \ size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name, checker_function) diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 38415005..159e05fe 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -24,7 +24,7 @@ void BM_Counters_Simple(benchmark::State& state) { state.counters["foo"] = 1; state.counters["bar"] = 2 * state.iterations(); } -BENCHMARK(BM_Counters_Simple);//->ThreadRange(1, 32); +BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"iterations\": %int,$", MR_Next}, @@ -39,7 +39,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", [](ResultsCheckerEntry const& e) { double its = e.GetAs< double >("iterations"); CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); // check that the value of bar is within 0.1% of the expected value - CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. * its, 0.001 * its); + CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2.*its, 0.001); }); // ========================================================================= // @@ -55,10 +55,10 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { state.SetBytesProcessed(364); state.SetItemsProcessed(150); } -BENCHMARK(BM_Counters_WithBytesAndItemsPSec);//->ThreadRange(1, 32); +BENCHMARK(BM_Counters_WithBytesAndItemsPSec); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " - "bar=%hrfloat foo=%hrfloat +%floatB/s +%float items/s$"}}); + "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %int,$", MR_Next}, @@ -73,12 +73,12 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," "%csv_bytes_items_report,%float,%float$"}}); CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", [](ResultsCheckerEntry const& e) { + double t = e.DurationCPUTime(); // this (and not real time) is the time used CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); // check that the values are within 0.1% of the expected values - double t = e.DurationCPUTime(); // this (and not real time) is the time used - CHECK_RESULT_VALUE(e, double, "bytes_per_second", EQ_EPS, 364. / t, 0.001 * t); - CHECK_RESULT_VALUE(e, double, "items_per_second", EQ_EPS, 150. / t, 0.001 * t); + CHECK_RESULT_VALUE_EPS(e, "bytes_per_second", EQ_EPS, 364./t, 0.001); + CHECK_RESULT_VALUE_EPS(e, "items_per_second", EQ_EPS, 150./t, 0.001); }); // ========================================================================= // @@ -92,7 +92,7 @@ void BM_Counters_Rate(benchmark::State& state) { state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; } -BENCHMARK(BM_Counters_Rate);//->ThreadRange(1, 32); +BENCHMARK(BM_Counters_Rate); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"iterations\": %int,$", MR_Next}, @@ -105,10 +105,10 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}}); CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", [](ResultsCheckerEntry const& e) { - // check that the values are within 0.1% of the expected values double t = e.DurationCPUTime(); // this (and not real time) is the time used - CHECK_COUNTER_VALUE(e, double, "foo", EQ_EPS, 5. / t, 0.001 * t); - CHECK_COUNTER_VALUE(e, double, "bar", EQ_EPS, 2. / t, 0.001 * t); + // check that the values are within 0.1% of the expected values + CHECK_COUNTER_VALUE_EPS(e, "foo", EQ_EPS, 1./t, 0.001); + CHECK_COUNTER_VALUE_EPS(e, "bar", EQ_EPS, 2./t, 0.001); }); // ========================================================================= //