github-repos/benchmark
1
0
Fork 0
mirror of https://github.com/google/benchmark.git synced 2025-02-06 01:00:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

implemented Complexity for O(1)

Browse Source
This commit is contained in:
Ismael 2016-05-18 21:25:00 +02:00
parent 27f0baa190
commit b73dc22944
7 changed files with 145 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/benchmark/reporter.h
View File

@ -48,7 +48,10 @@ class BenchmarkReporter {
cpu_accumulated_time(0),
bytes_per_second(0),
items_per_second(0),
max_heapbytes_used(0) {}
max_heapbytes_used(0),
complexity(O_1),
arg1(0),
arg2(0) {}
std::string benchmark_name;
std::string report_label; // Empty if not set by benchmark.
@ -63,6 +66,11 @@ class BenchmarkReporter {
// This is set to 0.0 if memory tracing is not enabled.
double max_heapbytes_used;
// Keep track of arguments to compute asymptotic complexity
BigO complexity;
int arg1;
int arg2;
};
// Called once for every suite of benchmarks run.
@ -78,6 +86,12 @@ class BenchmarkReporter {
// Note that all the grouped benchmark runs should refer to the same
// benchmark, thus have the same name.
virtual void ReportRuns(const std::vector<Run>& report) = 0;
// Called once at the last instance of a benchmark range, gives information about
// asymptotic complexity and RMS.
// Note that all the benchmark runs in a range should refer to the same benchmark,
// thus have the same name.
virtual void ReportComplexity(const std::vector<Run>& complexity_reports) = 0;
// Called once and only once after ever group of benchmarks is run and
// reported.
@ -85,7 +99,8 @@ class BenchmarkReporter {
virtual ~BenchmarkReporter();
protected:
static void ComputeStats(std::vector<Run> const& reports, Run* mean, Run* stddev);
static void ComputeStats(const std::vector<Run> & reports, Run& mean, Run& stddev);
static void ComputeBigO(const std::vector<Run> & reports, Run& bigO, Run& rms);
static TimeUnitMultiplier GetTimeUnitAndMultiplier(TimeUnit unit);
};
@ -95,6 +110,7 @@ class ConsoleReporter : public BenchmarkReporter {
public:
virtual bool ReportContext(const Context& context);
virtual void ReportRuns(const std::vector<Run>& reports);
virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
protected:
virtual void PrintRunData(const Run& report);
@ -107,6 +123,7 @@ public:
JSONReporter() : first_report_(true) {}
virtual bool ReportContext(const Context& context);
virtual void ReportRuns(const std::vector<Run>& reports);
virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
virtual void Finalize();
private:
@ -119,6 +136,7 @@ class CSVReporter : public BenchmarkReporter {
public:
virtual bool ReportContext(const Context& context);
virtual void ReportRuns(const std::vector<Run>& reports);
virtual void ReportComplexity(const std::vector<Run>& complexity_reports);
private:
void PrintRunData(const Run& report);

23
src/benchmark.cc
View File

@ -291,6 +291,7 @@ struct Benchmark::Instance {
bool use_real_time;
bool use_manual_time;
BigO complexity;
bool last_benchmark_instance;
double min_time;
int threads; // Number of concurrent threads to use
bool multithreaded; // Is benchmark multi-threaded?
@ -414,6 +415,7 @@ bool BenchmarkFamilies::FindBenchmarks(
instance.min_time = family->min_time_;
instance.use_real_time = family->use_real_time_;
instance.use_manual_time = family->use_manual_time_;
instance.last_benchmark_instance = (args == family->args_.back());
instance.complexity = family->complexity_;
instance.threads = num_threads;
instance.multithreaded = !(family->thread_counts_.empty());
@ -697,7 +699,8 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
}
void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
BenchmarkReporter* br) EXCLUDES(GetBenchmarkLock()) {
BenchmarkReporter* br,
std::vector<BenchmarkReporter::Run>& complexity_reports) EXCLUDES(GetBenchmarkLock()) {
size_t iters = 1;
std::vector<BenchmarkReporter::Run> reports;
@ -795,7 +798,14 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
report.cpu_accumulated_time = cpu_accumulated_time;
report.bytes_per_second = bytes_per_second;
report.items_per_second = items_per_second;
report.arg1 = b.arg1;
report.arg2 = b.arg2;
report.complexity = b.complexity;
reports.push_back(report);
if(report.complexity != O_None)
complexity_reports.push_back(report);
break;
}
@ -819,6 +829,12 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
}
}
br->ReportRuns(reports);
if((b.complexity != O_None) && b.last_benchmark_instance) {
br->ReportComplexity(complexity_reports);
complexity_reports.clear();
}
if (b.multithreaded) {
for (std::thread& thread : pool)
thread.join();
@ -903,9 +919,12 @@ void RunMatchingBenchmarks(const std::string& spec,
context.cpu_scaling_enabled = CpuScalingEnabled();
context.name_field_width = name_field_width;
// Keep track of runing times of all instances of current benchmark
std::vector<BenchmarkReporter::Run> complexity_reports;
if (reporter->ReportContext(context)) {
for (const auto& benchmark : benchmarks) {
RunBenchmark(benchmark, reporter);
RunBenchmark(benchmark, reporter, complexity_reports);
}
}
}

17
src/console_reporter.cc
View File

@ -72,13 +72,28 @@ void ConsoleReporter::ReportRuns(const std::vector<Run>& reports) {
Run mean_data;
Run stddev_data;
BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
// Output using PrintRun.
PrintRunData(mean_data);
PrintRunData(stddev_data);
}
void ConsoleReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
if (complexity_reports.size() < 2) {
// We don't report asymptotic complexity data if there was a single run.
return;
}
Run bigO_data;
Run rms_data;
BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
// Output using PrintRun.
PrintRunData(bigO_data);
PrintRunData(rms_data);
}
void ConsoleReporter::PrintRunData(const Run& result) {
// Format bytes per second
std::string rate;

21
src/csv_reporter.cc
View File

@ -48,7 +48,7 @@ bool CSVReporter::ReportContext(const Context& context) {
return true;
}
void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
void CSVReporter::ReportRuns(const std::vector<Run> & reports) {
if (reports.empty()) {
return;
}
@ -57,7 +57,7 @@ void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
if (reports.size() >= 2) {
Run mean_data;
Run stddev_data;
BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
reports_cp.push_back(mean_data);
reports_cp.push_back(stddev_data);
}
@ -66,7 +66,22 @@ void CSVReporter::ReportRuns(std::vector<Run> const& reports) {
}
}
void CSVReporter::PrintRunData(Run const& run) {
void CSVReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
if (complexity_reports.size() < 2) {
// We don't report asymptotic complexity data if there was a single run.
return;
}
Run bigO_data;
Run rms_data;
BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
// Output using PrintRun.
PrintRunData(bigO_data);
PrintRunData(rms_data);
}
void CSVReporter::PrintRunData(const Run & run) {
double multiplier;
const char* timeLabel;
std::tie(timeLabel, multiplier) = GetTimeUnitAndMultiplier(run.time_unit);

17
src/json_reporter.cc
View File

@ -100,7 +100,7 @@ void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
if (reports.size() >= 2) {
Run mean_data;
Run stddev_data;
BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data);
BenchmarkReporter::ComputeStats(reports, mean_data, stddev_data);
reports_cp.push_back(mean_data);
reports_cp.push_back(stddev_data);
}
@ -115,6 +115,21 @@ void JSONReporter::ReportRuns(std::vector<Run> const& reports) {
}
}
void JSONReporter::ReportComplexity(const std::vector<Run> & complexity_reports) {
if (complexity_reports.size() < 2) {
// We don't report asymptotic complexity data if there was a single run.
return;
}
Run bigO_data;
Run rms_data;
BenchmarkReporter::ComputeBigO(complexity_reports, bigO_data, rms_data);
// Output using PrintRun.
PrintRunData(bigO_data);
PrintRunData(rms_data);
}
void JSONReporter::Finalize() {
// Close the list of benchmarks and the top level object.
std::cout << "\n ]\n}\n";

68
src/reporter.cc
View File

@ -24,7 +24,7 @@ namespace benchmark {
void BenchmarkReporter::ComputeStats(
const std::vector<Run>& reports,
Run* mean_data, Run* stddev_data) {
Run& mean_data, Run& stddev_data) {
CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports";
// Accumulators.
Stat1_d real_accumulated_time_stat;
@ -48,33 +48,69 @@ void BenchmarkReporter::ComputeStats(
}
// Get the data from the accumulator to BenchmarkReporter::Run's.
mean_data->benchmark_name = reports[0].benchmark_name + "_mean";
mean_data->iterations = run_iterations;
mean_data->real_accumulated_time = real_accumulated_time_stat.Mean() *
mean_data.benchmark_name = reports[0].benchmark_name + "_mean";
mean_data.iterations = run_iterations;
mean_data.real_accumulated_time = real_accumulated_time_stat.Mean() *
run_iterations;
mean_data->cpu_accumulated_time = cpu_accumulated_time_stat.Mean() *
mean_data.cpu_accumulated_time = cpu_accumulated_time_stat.Mean() *
run_iterations;
mean_data->bytes_per_second = bytes_per_second_stat.Mean();
mean_data->items_per_second = items_per_second_stat.Mean();
mean_data.bytes_per_second = bytes_per_second_stat.Mean();
mean_data.items_per_second = items_per_second_stat.Mean();
// Only add label to mean/stddev if it is same for all runs
mean_data->report_label = reports[0].report_label;
mean_data.report_label = reports[0].report_label;
for (std::size_t i = 1; i < reports.size(); i++) {
if (reports[i].report_label != reports[0].report_label) {
mean_data->report_label = "";
mean_data.report_label = "";
break;
}
}
stddev_data->benchmark_name = reports[0].benchmark_name + "_stddev";
stddev_data->report_label = mean_data->report_label;
stddev_data->iterations = 0;
stddev_data->real_accumulated_time =
stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev";
stddev_data.report_label = mean_data.report_label;
stddev_data.iterations = 0;
stddev_data.real_accumulated_time =
real_accumulated_time_stat.StdDev();
stddev_data->cpu_accumulated_time =
stddev_data.cpu_accumulated_time =
cpu_accumulated_time_stat.StdDev();
stddev_data.bytes_per_second = bytes_per_second_stat.StdDev();
stddev_data.items_per_second = items_per_second_stat.StdDev();
}
void BenchmarkReporter::ComputeBigO(
const std::vector<Run>& reports,
Run& bigO, Run& rms) {
CHECK(reports.size() >= 2) << "Cannot compute asymptotic complexity for less than 2 reports";
// Accumulators.
Stat1_d real_accumulated_time_stat;
Stat1_d cpu_accumulated_time_stat;
// Populate the accumulators.
for (Run const& run : reports) {
real_accumulated_time_stat +=
Stat1_d(run.real_accumulated_time/run.iterations, run.iterations);
cpu_accumulated_time_stat +=
Stat1_d(run.cpu_accumulated_time/run.iterations, run.iterations);
}
std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/'));
// Get the data from the accumulator to BenchmarkReporter::Run's.
bigO.benchmark_name = benchmark_name + "_BigO";
bigO.iterations = 0;
bigO.real_accumulated_time = real_accumulated_time_stat.Mean();
bigO.cpu_accumulated_time = cpu_accumulated_time_stat.Mean();
// Only add label to mean/stddev if it is same for all runs
bigO.report_label = reports[0].report_label;
rms.benchmark_name = benchmark_name + "_RMS";
rms.report_label = bigO.report_label;
rms.iterations = 0;
rms.real_accumulated_time =
real_accumulated_time_stat.StdDev();
rms.cpu_accumulated_time =
cpu_accumulated_time_stat.StdDev();
stddev_data->bytes_per_second = bytes_per_second_stat.StdDev();
stddev_data->items_per_second = items_per_second_stat.StdDev();
}
TimeUnitMultiplier BenchmarkReporter::GetTimeUnitAndMultiplier(TimeUnit unit) {

4
test/complexity_test.cc
View File

@ -101,7 +101,7 @@ void BM_Extreme_Cases(benchmark::State& state) {
while (state.KeepRunning()) {
}
}
BENCHMARK(BM_Extreme_Cases);
BENCHMARK(BM_Extreme_Cases)->Arg(42);
BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N);
BENCHMARK(BM_Extreme_Cases)->Arg(42) -> Complexity(benchmark::O_Auto);
BENCHMARK_MAIN()