Merge pull request #108 from google/list-tests

Add --benchmark_list_tests option and change filter to match generated name.

src/benchmark.cc

@@ -40,6 +40,10 @@
 #include "sysinfo.h"
 #include "walltime.h"
 
+DEFINE_bool(benchmark_list_tests, false,
+            "Print a list of benchmarks. This option overrides all other "
+            "options.");
+
 DEFINE_string(benchmark_filter, ".",
               "A regular expression that specifies the set of benchmarks "
               "to execute.  If this flag is empty, no benchmarks are run.  "
@@ -373,7 +377,7 @@ bool BenchmarkFamilies::FindBenchmarks(
   MutexLock l(mutex_);
   for (BenchmarkImp* family : families_) {
     // Family was deleted or benchmark doesn't match
-    if (family == nullptr || !re.Match(family->name_)) continue;
+    if (family == nullptr) continue;
 
     if (family->arg_count_ == -1) {
       family->arg_count_ = 0;
@@ -417,7 +421,9 @@ bool BenchmarkFamilies::FindBenchmarks(
           instance.name += StringPrintF("/threads:%d", instance.threads);
         }
 
-        benchmarks->push_back(instance);
+        if (re.Match(instance.name)) {
+          benchmarks->push_back(instance);
+        }
       }
     }
   }
@@ -769,19 +775,30 @@ void State::SetLabel(const char* label) {
 }
 
 namespace internal {
+namespace {
+
+void PrintBenchmarkList() {
+  std::vector<Benchmark::Instance> benchmarks;
+  auto families = BenchmarkFamilies::GetInstance();
+  if (!families->FindBenchmarks(".", &benchmarks)) return;
+
+  for (const internal::Benchmark::Instance& benchmark : benchmarks) {
+    std::cout <<  benchmark.name << "\n";
+  }
+}
 
 void RunMatchingBenchmarks(const std::string& spec,
                            BenchmarkReporter* reporter) {
   CHECK(reporter != nullptr);
   if (spec.empty()) return;
 
-  std::vector<benchmark::internal::Benchmark::Instance> benchmarks;
-  auto families = benchmark::internal::BenchmarkFamilies::GetInstance();
+  std::vector<Benchmark::Instance> benchmarks;
+  auto families = BenchmarkFamilies::GetInstance();
   if (!families->FindBenchmarks(spec, &benchmarks)) return;
 
   // Determine the width of the name field using a minimum width of 10.
   size_t name_field_width = 10;
-  for (const internal::Benchmark::Instance& benchmark : benchmarks) {
+  for (const Benchmark::Instance& benchmark : benchmarks) {
     name_field_width =
         std::max<size_t>(name_field_width, benchmark.name.size());
   }
@@ -817,6 +834,7 @@ std::unique_ptr<BenchmarkReporter> GetDefaultReporter() {
   }
 }
 
+} // end namespace
 } // end namespace internal
 
 void RunSpecifiedBenchmarks() {
@@ -824,6 +842,10 @@ void RunSpecifiedBenchmarks() {
 }
 
 void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) {
+  if (FLAGS_benchmark_list_tests) {
+    internal::PrintBenchmarkList();
+    return;
+  }
   std::string spec = FLAGS_benchmark_filter;
   if (spec.empty() || spec == "all")
     spec = ".";  // Regexp that matches all benchmarks
@@ -842,7 +864,8 @@ namespace internal {
 void PrintUsageAndExit() {
   fprintf(stdout,
           "benchmark"
-          " [--benchmark_filter=<regex>]\n"
+          " [--benchmark_list_tests={true|false}]\n"
+          "          [--benchmark_filter=<regex>]\n"
           "          [--benchmark_min_time=<min_time>]\n"
           "          [--benchmark_repetitions=<num_repetitions>]\n"
           "          [--benchmark_format=<tabular|json|csv>]\n"
@@ -855,6 +878,8 @@ void ParseCommandLineFlags(int* argc, const char** argv) {
   using namespace benchmark;
   for (int i = 1; i < *argc; ++i) {
     if (
+        ParseBoolFlag(argv[i], "benchmark_list_tests",
+                      &FLAGS_benchmark_list_tests) ||
         ParseStringFlag(argv[i], "benchmark_filter",
                         &FLAGS_benchmark_filter) ||
         ParseDoubleFlag(argv[i], "benchmark_min_time",

test/CMakeLists.txt

@@ -13,25 +13,30 @@ endmacro(compile_benchmark_test)
 
 # Demonstration executable
 compile_benchmark_test(benchmark_test)
-add_test(benchmark benchmark_test --benchmark_min_time=0.1)
+add_test(benchmark benchmark_test --benchmark_min_time=0.01)
 
 compile_benchmark_test(filter_test)
-add_test(filter_simple filter_test --benchmark_filter=Calculate 16)
-add_test(filter_suffix filter_test --benchmark_filter=Calculate* 16)
-add_test(filter_regex_all filter_test --benchmark_filter=.* 16)
-add_test(filter_regex_blank filter_test --benchmark_filter= 16)
-add_test(filter_regex_none filter_test --benchmark_filter=monkey 0)
-add_test(filter_regex_wildcard filter_test --benchmark_filter=.*Calculate.* 16)
-add_test(filter_regex_begin filter_test --benchmark_filter=^BM_Calculate.* 16)
-add_test(filter_regex_end filter_test --benchmark_filter=.*Pi$ 8)
+macro(add_filter_test name filter expect)
+  add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect})
+endmacro(add_filter_test)
+
+add_filter_test(filter_simple "Foo" 3)
+add_filter_test(filter_suffix "BM_.*" 4)
+add_filter_test(filter_regex_all ".*" 5)
+add_filter_test(filter_regex_blank "" 5)
+add_filter_test(filter_regex_none "monkey" 0)
+add_filter_test(filter_regex_wildcard ".*Foo.*" 3)
+add_filter_test(filter_regex_begin "^BM_.*" 4)
+add_filter_test(filter_regex_begin2 "^N" 1)
+add_filter_test(filter_regex_end ".*Ba$" 1)
 
 compile_benchmark_test(options_test)
-add_test(options_benchmarks options_test)
+add_test(options_benchmarks options_test --benchmark_min_time=0.01)
 
 compile_benchmark_test(basic_test)
-add_test(basic_benchmark basic_test)
+add_test(basic_benchmark basic_test --benchmark_min_time=0.01)
 
 compile_benchmark_test(cxx03_test)
 set_target_properties(cxx03_test
     PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}")
-add_test(cxx03 cxx03_test)
+add_test(cxx03 cxx03_test --benchmark_min_time=0.01)

test/filter_test.cc

@@ -11,16 +11,6 @@
 
 namespace {
 
-double CalculatePi(int depth) {
-  double pi = 0.0;
-  for (int i = 0; i < depth; ++i) {
-    double numerator = static_cast<double>(((i % 2) * 2) - 1);
-    double denominator = static_cast<double>((2 * i) - 1);
-    pi += numerator / denominator;
-  }
-  return (pi - 1.0) * 4;
-}
-
 class TestReporter : public benchmark::ConsoleReporter {
  public:
   virtual bool ReportContext(const Context& context) {
@@ -46,32 +36,40 @@ class TestReporter : public benchmark::ConsoleReporter {
 
 }  // end namespace
 
-static void BM_CalculatePiRange(benchmark::State& state) {
-  double pi = 0.0;
-  while (state.KeepRunning())
-    pi = CalculatePi(state.range_x());
-  std::stringstream ss;
-  ss << pi;
-  state.SetLabel(ss.str());
-}
-BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024);
 
-static void BM_CalculatePi(benchmark::State& state) {
-  static const int depth = 1024;
-  double pi BENCHMARK_UNUSED = 0.0;
-  while (state.KeepRunning()) {
-    pi = CalculatePi(depth);
-  }
+static void NoPrefix(benchmark::State& state) {
+  while (state.KeepRunning()) {}
 }
-BENCHMARK(BM_CalculatePi)->Threads(8);
-BENCHMARK(BM_CalculatePi)->ThreadRange(1, 32);
-BENCHMARK(BM_CalculatePi)->ThreadPerCpu();
+BENCHMARK(NoPrefix);
+
+static void BM_Foo(benchmark::State& state) {
+  while (state.KeepRunning()) {}
+}
+BENCHMARK(BM_Foo);
+
+
+static void BM_Bar(benchmark::State& state) {
+  while (state.KeepRunning()) {}
+}
+BENCHMARK(BM_Bar);
+
+
+static void BM_FooBar(benchmark::State& state) {
+  while (state.KeepRunning()) {}
+}
+BENCHMARK(BM_FooBar);
+
+
+static void BM_FooBa(benchmark::State& state) {
+  while (state.KeepRunning()) {}
+}
+BENCHMARK(BM_FooBa);
+
+
 
 int main(int argc, const char* argv[]) {
   benchmark::Initialize(&argc, argv);
 
-  assert(std::fabs(CalculatePi(1)) < std::numeric_limits<float>::epsilon());
-
   TestReporter test_reporter;
   benchmark::RunSpecifiedBenchmarks(&test_reporter);