From b73dc22944cb933289bbdbf5bb6616dbfc50168f Mon Sep 17 00:00:00 2001 From: Ismael Date: Wed, 18 May 2016 21:25:00 +0200 Subject: [PATCH] implemented Complexity for O(1) --- include/benchmark/reporter.h | 22 ++++++++++-- src/benchmark.cc | 23 ++++++++++-- src/console_reporter.cc | 17 ++++++++- src/csv_reporter.cc | 21 +++++++++-- src/json_reporter.cc | 17 ++++++++- src/reporter.cc | 68 +++++++++++++++++++++++++++--------- test/complexity_test.cc | 4 +-- 7 files changed, 145 insertions(+), 27 deletions(-) diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index aaf5fbff..b3988002 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -48,7 +48,10 @@ class BenchmarkReporter { cpu_accumulated_time(0), bytes_per_second(0), items_per_second(0), - max_heapbytes_used(0) {} + max_heapbytes_used(0), + complexity(O_1), + arg1(0), + arg2(0) {} std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. @@ -63,6 +66,11 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; + + // Keep track of arguments to compute asymptotic complexity + BigO complexity; + int arg1; + int arg2; }; // Called once for every suite of benchmarks run. @@ -78,6 +86,12 @@ class BenchmarkReporter { // Note that all the grouped benchmark runs should refer to the same // benchmark, thus have the same name. virtual void ReportRuns(const std::vector& report) = 0; + + // Called once at the last instance of a benchmark range, gives information about + // asymptotic complexity and RMS. + // Note that all the benchmark runs in a range should refer to the same benchmark, + // thus have the same name. + virtual void ReportComplexity(const std::vector& complexity_reports) = 0; // Called once and only once after ever group of benchmarks is run and // reported. @@ -85,7 +99,8 @@ class BenchmarkReporter { virtual ~BenchmarkReporter(); protected: - static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); + static void ComputeStats(const std::vector & reports, Run& mean, Run& stddev); + static void ComputeBigO(const std::vector & reports, Run& bigO, Run& rms); static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit); }; @@ -95,6 +110,7 @@ class ConsoleReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); protected: virtual void PrintRunData(const Run& report); @@ -107,6 +123,7 @@ public: JSONReporter() : first_report_(true) {} virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); virtual void Finalize(); private: @@ -119,6 +136,7 @@ class CSVReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); + virtual void ReportComplexity(const std::vector& complexity_reports); private: void PrintRunData(const Run& report); diff --git a/src/benchmark.cc b/src/benchmark.cc index afe1e441..874dc0c7 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -291,6 +291,7 @@ struct Benchmark::Instance { bool use_real_time; bool use_manual_time; BigO complexity; + bool last_benchmark_instance; double min_time; int threads; // Number of concurrent threads to use bool multithreaded; // Is benchmark multi-threaded? @@ -414,6 +415,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; + instance.last_benchmark_instance = (args == family->args_.back()); instance.complexity = family->complexity_; instance.threads = num_threads; instance.multithreaded = !(family->thread_counts_.empty()); @@ -697,7 +699,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, } void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, - BenchmarkReporter* br) EXCLUDES(GetBenchmarkLock()) { + BenchmarkReporter* br, + std::vector& complexity_reports) EXCLUDES(GetBenchmarkLock()) { size_t iters = 1; std::vector reports; @@ -795,7 +798,14 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.cpu_accumulated_time = cpu_accumulated_time; report.bytes_per_second = bytes_per_second; report.items_per_second = items_per_second; + report.arg1 = b.arg1; + report.arg2 = b.arg2; + report.complexity = b.complexity; reports.push_back(report); + + if(report.complexity != O_None) + complexity_reports.push_back(report); + break; } @@ -819,6 +829,12 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, } } br->ReportRuns(reports); + + if((b.complexity != O_None) && b.last_benchmark_instance) { + br->ReportComplexity(complexity_reports); + complexity_reports.clear(); + } + if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); @@ -903,9 +919,12 @@ void RunMatchingBenchmarks(const std::string& spec, context.cpu_scaling_enabled = CpuScalingEnabled(); context.name_field_width = name_field_width; + // Keep track of runing times of all instances of current benchmark + std::vector complexity_reports; + if (reporter->ReportContext(context)) { for (const auto& benchmark : benchmarks) { - RunBenchmark(benchmark, reporter); + RunBenchmark(benchmark, reporter, complexity_reports); } } } diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 56bd3ced..0d8ab1dc 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -72,13 +72,28 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); // Output using PrintRun. PrintRunData(mean_data); PrintRunData(stddev_data); } +void ConsoleReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + void ConsoleReporter::PrintRunData(const Run& result) { // Format bytes per second std::string rate; diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 3f67d1de..f13a5f8b 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -48,7 +48,7 @@ bool CSVReporter::ReportContext(const Context& context) { return true; } -void CSVReporter::ReportRuns(std::vector const& reports) { +void CSVReporter::ReportRuns(const std::vector & reports) { if (reports.empty()) { return; } @@ -57,7 +57,7 @@ void CSVReporter::ReportRuns(std::vector const& reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -66,7 +66,22 @@ void CSVReporter::ReportRuns(std::vector const& reports) { } } -void CSVReporter::PrintRunData(Run const& run) { +void CSVReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + +void CSVReporter::PrintRunData(const Run & run) { double multiplier; const char* timeLabel; std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit); diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 7ed141fc..07fc3662 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -100,7 +100,7 @@ void JSONReporter::ReportRuns(std::vector const& reports) { if (reports.size() >= 2) { Run mean_data; Run stddev_data; - BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data); reports_cp.push_back(mean_data); reports_cp.push_back(stddev_data); } @@ -115,6 +115,21 @@ void JSONReporter::ReportRuns(std::vector const& reports) { } } +void JSONReporter::ReportComplexity(const std::vector & complexity_reports) { + if (complexity_reports.size() < 2) { + // We don't report asymptotic complexity data if there was a single run. + return; + } + + Run bigO_data; + Run rms_data; + BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data); + + // Output using PrintRun. + PrintRunData(bigO_data); + PrintRunData(rms_data); +} + void JSONReporter::Finalize() { // Close the list of benchmarks and the top level object. std::cout << "\n ]\n}\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 036546e7..fd97aba4 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -24,7 +24,7 @@ namespace benchmark { void BenchmarkReporter::ComputeStats( const std::vector& reports, - Run* mean_data, Run* stddev_data) { + Run& mean_data, Run& stddev_data) { CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports"; // Accumulators. Stat1_d real_accumulated_time_stat; @@ -48,33 +48,69 @@ void BenchmarkReporter::ComputeStats( } // Get the data from the accumulator to BenchmarkReporter::Run's. - mean_data->benchmark_name = reports[0].benchmark_name + "_mean"; - mean_data->iterations = run_iterations; - mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() * + mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; + mean_data.iterations = run_iterations; + mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() * run_iterations; - mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * + mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() * run_iterations; - mean_data->bytes_per_second = bytes_per_second_stat.Mean(); - mean_data->items_per_second = items_per_second_stat.Mean(); + mean_data.bytes_per_second = bytes_per_second_stat.Mean(); + mean_data.items_per_second = items_per_second_stat.Mean(); // Only add label to mean/stddev if it is same for all runs - mean_data->report_label = reports[0].report_label; + mean_data.report_label = reports[0].report_label; for (std::size_t i = 1; i < reports.size(); i++) { if (reports[i].report_label != reports[0].report_label) { - mean_data->report_label = ""; + mean_data.report_label = ""; break; } } - stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev"; - stddev_data->report_label = mean_data->report_label; - stddev_data->iterations = 0; - stddev_data->real_accumulated_time = + stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; + stddev_data.report_label = mean_data.report_label; + stddev_data.iterations = 0; + stddev_data.real_accumulated_time = real_accumulated_time_stat.StdDev(); - stddev_data->cpu_accumulated_time = + stddev_data.cpu_accumulated_time = + cpu_accumulated_time_stat.StdDev(); + stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); + stddev_data.items_per_second = items_per_second_stat.StdDev(); +} + +void BenchmarkReporter::ComputeBigO( + const std::vector& reports, + Run& bigO, Run& rms) { + CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports"; + // Accumulators. + Stat1_d real_accumulated_time_stat; + Stat1_d cpu_accumulated_time_stat; + + // Populate the accumulators. + for (Run const& run : reports) { + real_accumulated_time_stat += + Stat1_d(run.real_accumulated_time/run.iterations, run.iterations); + cpu_accumulated_time_stat += + Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations); + } + + std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + + // Get the data from the accumulator to BenchmarkReporter::Run's. + bigO.benchmark_name = benchmark_name + "_BigO"; + bigO.iterations = 0; + bigO.real_accumulated_time = real_accumulated_time_stat.Mean(); + bigO.cpu_accumulated_time = cpu_accumulated_time_stat.Mean(); + + // Only add label to mean/stddev if it is same for all runs + bigO.report_label = reports[0].report_label; + + rms.benchmark_name = benchmark_name + "_RMS"; + rms.report_label = bigO.report_label; + rms.iterations = 0; + rms.real_accumulated_time = + real_accumulated_time_stat.StdDev(); + rms.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); - stddev_data->bytes_per_second = bytes_per_second_stat.StdDev(); - stddev_data->items_per_second = items_per_second_stat.StdDev(); } TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) { diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 777c0f39..afa82edb 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -101,7 +101,7 @@ void BM_Extreme_Cases(benchmark::State& state) { while (state.KeepRunning()) { } } -BENCHMARK(BM_Extreme_Cases); -BENCHMARK(BM_Extreme_Cases)->Arg(42); +BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N); +BENCHMARK(BM_Extreme_Cases)->Arg(42) -> Complexity(benchmark::O_Auto); BENCHMARK_MAIN() \ No newline at end of file