implemented Complexity for O(1)

include/benchmark/reporter.h

@@ -48,7 +48,10 @@ class BenchmarkReporter {
       cpu_accumulated_time(0),
       bytes_per_second(0),
       items_per_second(0),
-      max_heapbytes_used(0) {}
+      max_heapbytes_used(0),
+      complexity(O_1),
+      arg1(0), 
+      arg2(0) {}
 
     std::string benchmark_name;
     std::string report_label;  // Empty if not set by benchmark.
@@ -63,6 +66,11 @@ class BenchmarkReporter {
 
     // This is set to 0.0 if memory tracing is not enabled.
     double max_heapbytes_used;
+    
+    // Keep track of arguments to compute asymptotic complexity
+    BigO   complexity;
+    int    arg1;
+    int    arg2;
   };
 
   // Called once for every suite of benchmarks run.
@@ -78,6 +86,12 @@ class BenchmarkReporter {
   // Note that all the grouped benchmark runs should refer to the same
   // benchmark, thus have the same name.
   virtual void ReportRuns(const std::vector<Run>& report) = 0;
+  
+  // Called once at the last instance of a benchmark range, gives information about
+  // asymptotic complexity and RMS. 
+  // Note that all the benchmark runs in a range should refer to the same benchmark, 
+  // thus have the same name.
+  virtual void ReportComplexity(const std::vector<Run>& complexity_reports) = 0;
 
   // Called once and only once after ever group of benchmarks is run and
   // reported.
@@ -85,7 +99,8 @@ class BenchmarkReporter {
 
   virtual ~BenchmarkReporter();
 protected:
-  static void ComputeStats(std::vector<Run> const& reports, Run* mean, Run* stddev);
+  static void ComputeStats(const std::vector<Run> & reports, Run& mean, Run& stddev);
+  static void ComputeBigO(const std::vector<Run> & reports, Run& bigO, Run& rms);
   static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit);
 };
 
@@ -95,6 +110,7 @@ class ConsoleReporter : public BenchmarkReporter {
  public:
   virtual bool ReportContext(const Context& context);
   virtual void ReportRuns(const std::vector<Run>& reports);
+  virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
 
  protected:
   virtual void PrintRunData(const Run& report);
@@ -107,6 +123,7 @@ public:
   JSONReporter() : first_report_(true) {}
   virtual bool ReportContext(const Context& context);
   virtual void ReportRuns(const std::vector<Run>& reports);
+  virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
   virtual void Finalize();
 
 private:
@@ -119,6 +136,7 @@ class CSVReporter : public BenchmarkReporter {
 public:
   virtual bool ReportContext(const Context& context);
   virtual void ReportRuns(const std::vector<Run>& reports);
+  virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
 
 private:
   void PrintRunData(const Run& report);

src/benchmark.cc

@@ -291,6 +291,7 @@ struct Benchmark::Instance {
   bool           use_real_time;
   bool           use_manual_time;
   BigO           complexity;
+  bool           last_benchmark_instance;
   double         min_time;
   int            threads;    // Number of concurrent threads to use
   bool           multithreaded;  // Is benchmark multi-threaded?
@@ -414,6 +415,7 @@ bool BenchmarkFamilies::FindBenchmarks(
         instance.min_time = family->min_time_;
         instance.use_real_time = family->use_real_time_;
         instance.use_manual_time = family->use_manual_time_;
+        instance.last_benchmark_instance = (args == family->args_.back());
         instance.complexity = family->complexity_;
         instance.threads = num_threads;
         instance.multithreaded = !(family->thread_counts_.empty());
@@ -697,7 +699,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
 }
 
 void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
-                  BenchmarkReporter* br) EXCLUDES(GetBenchmarkLock()) {
+                  BenchmarkReporter* br,
+                  std::vector<BenchmarkReporter::Run>& complexity_reports) EXCLUDES(GetBenchmarkLock()) {
   size_t iters = 1;
 
   std::vector<BenchmarkReporter::Run> reports;
@@ -795,7 +798,14 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
         report.cpu_accumulated_time = cpu_accumulated_time;
         report.bytes_per_second = bytes_per_second;
         report.items_per_second = items_per_second;
+        report.arg1 = b.arg1;
+        report.arg2 = b.arg2;
+        report.complexity = b.complexity;
         reports.push_back(report);
+        
+        if(report.complexity != O_None) 
+          complexity_reports.push_back(report);
+     
         break;
       }
 
@@ -819,6 +829,12 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
     }
   }
   br->ReportRuns(reports);
+  
+  if((b.complexity != O_None) && b.last_benchmark_instance) {
+    br->ReportComplexity(complexity_reports);
+    complexity_reports.clear();
+  }
+  
   if (b.multithreaded) {
     for (std::thread& thread : pool)
       thread.join();
@@ -903,9 +919,12 @@ void RunMatchingBenchmarks(const std::string& spec,
   context.cpu_scaling_enabled = CpuScalingEnabled();
   context.name_field_width = name_field_width;
 
+  // Keep track of runing times of all instances of current benchmark
+  std::vector<BenchmarkReporter::Run> complexity_reports;
+
   if (reporter->ReportContext(context)) {
     for (const auto& benchmark : benchmarks) {
-      RunBenchmark(benchmark, reporter);
+      RunBenchmark(benchmark, reporter, complexity_reports);
     }
   }
 }

src/console_reporter.cc

@@ -72,13 +72,28 @@ void ConsoleReporter::ReportRuns(const std::vector<Run>& reports) {
 
   Run mean_data;
   Run stddev_data;
-  BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
+  BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
 
   // Output using PrintRun.
   PrintRunData(mean_data);
   PrintRunData(stddev_data);
 }
 
+void ConsoleReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
+  if (complexity_reports.size() < 2) {
+    // We don't report asymptotic complexity data if there was a single run.
+    return;
+  }
+  
+  Run bigO_data;
+  Run rms_data;
+  BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
+  
+  // Output using PrintRun.
+  PrintRunData(bigO_data);
+  PrintRunData(rms_data);
+}
+
 void ConsoleReporter::PrintRunData(const Run& result) {
   // Format bytes per second
   std::string rate;

src/csv_reporter.cc

@@ -48,7 +48,7 @@ bool CSVReporter::ReportContext(const Context& context) {
   return true;
 }
 
-void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
+void CSVReporter::ReportRuns(const std::vector<Run> & reports) {
   if (reports.empty()) {
     return;
   }
@@ -57,7 +57,7 @@ void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
   if (reports.size() >= 2) {
     Run mean_data;
     Run stddev_data;
-    BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
+    BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
     reports_cp.push_back(mean_data);
     reports_cp.push_back(stddev_data);
   }
@@ -66,7 +66,22 @@ void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
   }
 }
 
-void CSVReporter::PrintRunData(Run const& run) {
+void CSVReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
+  if (complexity_reports.size() < 2) {
+    // We don't report asymptotic complexity data if there was a single run.
+    return;
+  }
+  
+  Run bigO_data;
+  Run rms_data;
+  BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
+  
+  // Output using PrintRun.
+  PrintRunData(bigO_data);
+  PrintRunData(rms_data);
+}
+
+void CSVReporter::PrintRunData(const Run & run) {
   double multiplier;
   const char* timeLabel;
   std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit);

src/json_reporter.cc

@@ -100,7 +100,7 @@ void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
   if (reports.size() >= 2) {
     Run mean_data;
     Run stddev_data;
-    BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
+    BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
     reports_cp.push_back(mean_data);
     reports_cp.push_back(stddev_data);
   }
@@ -115,6 +115,21 @@ void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
   }
 }
 
+void JSONReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
+  if (complexity_reports.size() < 2) {
+    // We don't report asymptotic complexity data if there was a single run.
+    return;
+  }
+  
+  Run bigO_data;
+  Run rms_data;
+  BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
+  
+  // Output using PrintRun.
+  PrintRunData(bigO_data);
+  PrintRunData(rms_data);
+}
+
 void JSONReporter::Finalize() {
     // Close the list of benchmarks and the top level object.
     std::cout << "\n  ]\n}\n";

src/reporter.cc

@@ -24,7 +24,7 @@ namespace benchmark {
 
 void BenchmarkReporter::ComputeStats(
     const std::vector<Run>& reports,
-    Run* mean_data, Run* stddev_data) {
+    Run& mean_data, Run& stddev_data) {
   CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports";
   // Accumulators.
   Stat1_d real_accumulated_time_stat;
@@ -48,33 +48,69 @@ void BenchmarkReporter::ComputeStats(
   }
 
   // Get the data from the accumulator to BenchmarkReporter::Run's.
-  mean_data->benchmark_name = reports[0].benchmark_name + "_mean";
+  mean_data.benchmark_name = reports[0].benchmark_name + "_mean";
-  mean_data->iterations = run_iterations;
+  mean_data.iterations = run_iterations;
-  mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() *
+  mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() *
                                      run_iterations;
-  mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() *
+  mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() *
                                     run_iterations;
-  mean_data->bytes_per_second = bytes_per_second_stat.Mean();
+  mean_data.bytes_per_second = bytes_per_second_stat.Mean();
-  mean_data->items_per_second = items_per_second_stat.Mean();
+  mean_data.items_per_second = items_per_second_stat.Mean();
 
   // Only add label to mean/stddev if it is same for all runs
-  mean_data->report_label = reports[0].report_label;
+  mean_data.report_label = reports[0].report_label;
   for (std::size_t i = 1; i < reports.size(); i++) {
     if (reports[i].report_label != reports[0].report_label) {
-      mean_data->report_label = "";
+      mean_data.report_label = "";
       break;
     }
   }
 
-  stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev";
+  stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev";
-  stddev_data->report_label = mean_data->report_label;
+  stddev_data.report_label = mean_data.report_label;
-  stddev_data->iterations = 0;
+  stddev_data.iterations = 0;
-  stddev_data->real_accumulated_time =
+  stddev_data.real_accumulated_time =
       real_accumulated_time_stat.StdDev();
-  stddev_data->cpu_accumulated_time =
+  stddev_data.cpu_accumulated_time =
+      cpu_accumulated_time_stat.StdDev();
+  stddev_data.bytes_per_second = bytes_per_second_stat.StdDev();
+  stddev_data.items_per_second = items_per_second_stat.StdDev();
+}
+
+void BenchmarkReporter::ComputeBigO(
+    const std::vector<Run>& reports,
+    Run& bigO, Run& rms) {
+  CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports";
+  // Accumulators.
+  Stat1_d real_accumulated_time_stat;
+  Stat1_d cpu_accumulated_time_stat;
+
+  // Populate the accumulators.
+  for (Run const& run : reports) {
+    real_accumulated_time_stat +=
+        Stat1_d(run.real_accumulated_time/run.iterations, run.iterations);
+    cpu_accumulated_time_stat +=
+        Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations);
+  }
+
+  std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/'));
+  
+  // Get the data from the accumulator to BenchmarkReporter::Run's.
+  bigO.benchmark_name = benchmark_name + "_BigO";
+  bigO.iterations = 0;
+  bigO.real_accumulated_time = real_accumulated_time_stat.Mean();
+  bigO.cpu_accumulated_time = cpu_accumulated_time_stat.Mean();
+
+  // Only add label to mean/stddev if it is same for all runs
+  bigO.report_label = reports[0].report_label;
+
+  rms.benchmark_name = benchmark_name + "_RMS";
+  rms.report_label = bigO.report_label;
+  rms.iterations = 0;
+  rms.real_accumulated_time =
+      real_accumulated_time_stat.StdDev();
+  rms.cpu_accumulated_time =
       cpu_accumulated_time_stat.StdDev();
-  stddev_data->bytes_per_second = bytes_per_second_stat.StdDev();
-  stddev_data->items_per_second = items_per_second_stat.StdDev();
 }
 
 TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) {

test/complexity_test.cc

@@ -101,7 +101,7 @@ void BM_Extreme_Cases(benchmark::State& state) {
   while (state.KeepRunning()) {
   }
 }
-BENCHMARK(BM_Extreme_Cases);
+BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N);
-BENCHMARK(BM_Extreme_Cases)->Arg(42);
+BENCHMARK(BM_Extreme_Cases)->Arg(42) -> Complexity(benchmark::O_Auto);
 
 BENCHMARK_MAIN()