refactor MinimalLEastSq

This commit is contained in:
Ismael 2016-05-23 20:12:54 +02:00
parent 5e52d2d6c0
commit ac05c04533
10 changed files with 91 additions and 92 deletions

View File

@ -127,14 +127,14 @@ static void BM_StringCompare(benchmark::State& state) {
benchmark::DoNotOptimize(s1.compare(s2)); benchmark::DoNotOptimize(s1.compare(s2));
} }
BENCHMARK(BM_StringCompare) BENCHMARK(BM_StringCompare)
->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_N); ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);
``` ```
As shown in the following invocation, asymptotic complexity might also be calculated automatically. As shown in the following invocation, asymptotic complexity might also be calculated automatically.
```c++ ```c++
BENCHMARK(BM_StringCompare) BENCHMARK(BM_StringCompare)
->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::O_Auto); ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oAuto);
``` ```
### Templated benchmarks ### Templated benchmarks

View File

@ -232,17 +232,17 @@ enum TimeUnit {
}; };
// BigO is passed to a benchmark in order to specify the asymptotic computational // BigO is passed to a benchmark in order to specify the asymptotic computational
// complexity for the benchmark. In case O_Auto is selected, complexity will be // complexity for the benchmark. In case oAuto is selected, complexity will be
// calculated automatically to the best fit. // calculated automatically to the best fit.
enum BigO { enum BigO {
O_None, oNone,
O_1, o1,
O_N, oN,
O_N_Squared, oNSquared,
O_N_Cubed, oNCubed,
O_log_N, oLogN,
O_N_log_N, oNLogN,
O_Auto oAuto
}; };
// State is passed to a running Benchmark and contains state for the // State is passed to a running Benchmark and contains state for the

View File

@ -49,7 +49,7 @@ class BenchmarkReporter {
bytes_per_second(0), bytes_per_second(0),
items_per_second(0), items_per_second(0),
max_heapbytes_used(0), max_heapbytes_used(0),
complexity(O_None), complexity(oNone),
arg1(0), arg1(0),
arg2(0), arg2(0),
report_big_o(false), report_big_o(false),

View File

@ -454,7 +454,7 @@ BenchmarkImp::BenchmarkImp(const char* name)
: name_(name), arg_count_(-1), time_unit_(kNanosecond), : name_(name), arg_count_(-1), time_unit_(kNanosecond),
range_multiplier_(kRangeMultiplier), min_time_(0.0), range_multiplier_(kRangeMultiplier), min_time_(0.0),
use_real_time_(false), use_manual_time_(false), use_real_time_(false), use_manual_time_(false),
complexity_(O_None) { complexity_(oNone) {
} }
BenchmarkImp::~BenchmarkImp() { BenchmarkImp::~BenchmarkImp() {
@ -803,7 +803,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
report.complexity = b.complexity; report.complexity = b.complexity;
reports.push_back(report); reports.push_back(report);
if(report.complexity != O_None) if(report.complexity != oNone)
complexity_reports.push_back(report); complexity_reports.push_back(report);
break; break;
@ -830,7 +830,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
} }
br->ReportRuns(reports); br->ReportRuns(reports);
if((b.complexity != O_None) && b.last_benchmark_instance) { if((b.complexity != oNone) && b.last_benchmark_instance) {
br->ReportComplexity(complexity_reports); br->ReportComplexity(complexity_reports);
complexity_reports.clear(); complexity_reports.clear();
} }

View File

@ -72,12 +72,12 @@ void CSVReporter::ReportComplexity(const std::vector<Run> & complexity_reports)
return; return;
} }
Run bigO_data; Run big_o_data;
Run rms_data; Run rms_data;
BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data);
// Output using PrintRun. // Output using PrintRun.
PrintRunData(bigO_data); PrintRunData(big_o_data);
PrintRunData(rms_data); PrintRunData(rms_data);
} }

View File

@ -127,13 +127,13 @@ void JSONReporter::ReportComplexity(const std::vector<Run> & complexity_reports)
out << ",\n"; out << ",\n";
} }
Run bigO_data; Run big_o_data;
Run rms_data; Run rms_data;
BenchmarkReporter::ComputeBigO(complexity_reports, &bigO_data, &rms_data); BenchmarkReporter::ComputeBigO(complexity_reports, &big_o_data, &rms_data);
// Output using PrintRun. // Output using PrintRun.
out << indent << "{\n"; out << indent << "{\n";
PrintRunData(bigO_data); PrintRunData(big_o_data);
out << indent << "},\n"; out << indent << "},\n";
out << indent << "{\n"; out << indent << "{\n";
PrintRunData(rms_data); PrintRunData(rms_data);

View File

@ -16,95 +16,94 @@
// Adapted to be used with google benchmark // Adapted to be used with google benchmark
#include "minimal_leastsq.h" #include "minimal_leastsq.h"
#include "check.h"
#include <math.h> #include <math.h>
// Internal function to calculate the different scalability forms // Internal function to calculate the different scalability forms
double fittingCurve(double n, benchmark::BigO complexity) { double FittingCurve(double n, benchmark::BigO complexity) {
switch (complexity) { switch (complexity) {
case benchmark::O_N: case benchmark::oN:
return n; return n;
case benchmark::O_N_Squared: case benchmark::oNSquared:
return pow(n, 2); return pow(n, 2);
case benchmark::O_N_Cubed: case benchmark::oNCubed:
return pow(n, 3); return pow(n, 3);
case benchmark::O_log_N: case benchmark::oLogN:
return log2(n); return log2(n);
case benchmark::O_N_log_N: case benchmark::oNLogN:
return n * log2(n); return n * log2(n);
case benchmark::O_1: case benchmark::o1:
default: default:
return 1; return 1;
} }
} }
// Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. // Internal function to find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error.
// - N : Vector containing the size of the benchmark tests. // - n : Vector containing the size of the benchmark tests.
// - Time : Vector containing the times for the benchmark tests. // - time : Vector containing the times for the benchmark tests.
// - Complexity : Fitting curve. // - complexity : Fitting curve.
// For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit // For a deeper explanation on the algorithm logic, look the README file at http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit
LeastSq leastSq(const std::vector<int>& N, const std::vector<double>& Time, const benchmark::BigO Complexity) { LeastSq CalculateLeastSq(const std::vector<int>& n, const std::vector<double>& time, const benchmark::BigO complexity) {
assert(N.size() == Time.size() && N.size() >= 2); CHECK_NE(complexity, benchmark::oAuto);
assert(Complexity != benchmark::O_None &&
Complexity != benchmark::O_Auto);
double sigmaGN = 0; double sigma_gn = 0;
double sigmaGNSquared = 0; double sigma_gn_squared = 0;
double sigmaTime = 0; double sigma_time = 0;
double sigmaTimeGN = 0; double sigma_time_gn = 0;
// Calculate least square fitting parameter // Calculate least square fitting parameter
for (size_t i = 0; i < N.size(); ++i) { for (size_t i = 0; i < n.size(); ++i) {
double GNi = fittingCurve(N[i], Complexity); double gn_i = FittingCurve(n[i], complexity);
sigmaGN += GNi; sigma_gn += gn_i;
sigmaGNSquared += GNi * GNi; sigma_gn_squared += gn_i * gn_i;
sigmaTime += Time[i]; sigma_time += time[i];
sigmaTimeGN += Time[i] * GNi; sigma_time_gn += time[i] * gn_i;
} }
LeastSq result; LeastSq result;
result.complexity = Complexity; result.complexity = complexity;
// Calculate complexity. // Calculate complexity.
// O_1 is treated as an special case // o1 is treated as an special case
if (Complexity != benchmark::O_1) if (complexity != benchmark::o1)
result.coef = sigmaTimeGN / sigmaGNSquared; result.coef = sigma_time_gn / sigma_gn_squared;
else else
result.coef = sigmaTime / N.size(); result.coef = sigma_time / n.size();
// Calculate RMS // Calculate RMS
double rms = 0; double rms = 0;
for (size_t i = 0; i < N.size(); ++i) { for (size_t i = 0; i < n.size(); ++i) {
double fit = result.coef * fittingCurve(N[i], Complexity); double fit = result.coef * FittingCurve(n[i], complexity);
rms += pow((Time[i] - fit), 2); rms += pow((time[i] - fit), 2);
} }
double mean = sigmaTime / N.size(); double mean = sigma_time / n.size();
result.rms = sqrt(rms / N.size()) / mean; // Normalized RMS by the mean of the observed values result.rms = sqrt(rms / n.size()) / mean; // Normalized RMS by the mean of the observed values
return result; return result;
} }
// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error.
// - N : Vector containing the size of the benchmark tests. // - n : Vector containing the size of the benchmark tests.
// - Time : Vector containing the times for the benchmark tests. // - time : Vector containing the times for the benchmark tests.
// - Complexity : If different than O_Auto, the fitting curve will stick to this one. If it is O_Auto, it will be calculated // - complexity : If different than oAuto, the fitting curve will stick to this one. If it is oAuto, it will be calculated
// the best fitting curve. // the best fitting curve.
LeastSq minimalLeastSq(const std::vector<int>& N, const std::vector<double>& Time, const benchmark::BigO Complexity) { LeastSq MinimalLeastSq(const std::vector<int>& n, const std::vector<double>& time, const benchmark::BigO complexity) {
assert(N.size() == Time.size() && N.size() >= 2); // Do not compute fitting curve is less than two benchmark runs are given CHECK_EQ(n.size(), time.size());
assert(Complexity != benchmark::O_None); // Check that complexity is a valid parameter. CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two benchmark runs are given
CHECK_NE(complexity, benchmark::oNone);
if(Complexity == benchmark::O_Auto) { if(complexity == benchmark::oAuto) {
std::vector<benchmark::BigO> fitCurves = { benchmark::O_log_N, benchmark::O_N, benchmark::O_N_log_N, benchmark::O_N_Squared, benchmark::O_N_Cubed }; std::vector<benchmark::BigO> fit_curves = { benchmark::oLogN, benchmark::oN, benchmark::oNLogN, benchmark::oNSquared, benchmark::oNCubed };
LeastSq best_fit = leastSq(N, Time, benchmark::O_1); // Take O_1 as default best fitting curve LeastSq best_fit = CalculateLeastSq(n, time, benchmark::o1); // Take o1 as default best fitting curve
// Compute all possible fitting curves and stick to the best one // Compute all possible fitting curves and stick to the best one
for (const auto& fit : fitCurves) { for (const auto& fit : fit_curves) {
LeastSq current_fit = leastSq(N, Time, fit); LeastSq current_fit = CalculateLeastSq(n, time, fit);
if (current_fit.rms < best_fit.rms) if (current_fit.rms < best_fit.rms)
best_fit = current_fit; best_fit = current_fit;
} }
@ -112,5 +111,5 @@ LeastSq minimalLeastSq(const std::vector<int>& N, const std::vector<double>& Tim
return best_fit; return best_fit;
} }
else else
return leastSq(N, Time, Complexity); return CalculateLeastSq(n, time, complexity);
} }

View File

@ -22,18 +22,18 @@
#include <vector> #include <vector>
// This data structure will contain the result returned by minimalLeastSq // This data structure will contain the result returned by MinimalLeastSq
// - coef : Estimated coeficient for the high-order term as interpolated from data. // - coef : Estimated coeficient for the high-order term as interpolated from data.
// - rms : Normalized Root Mean Squared Error. // - rms : Normalized Root Mean Squared Error.
// - complexity : Scalability form (e.g. O_N, O_N_log_N). In case a scalability form has been provided to minimalLeastSq // - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability form has been provided to MinimalLeastSq
// this will return the same value. In case BigO::O_Auto has been selected, this parameter will return the // this will return the same value. In case BigO::oAuto has been selected, this parameter will return the
// best fitting curve detected. // best fitting curve detected.
struct LeastSq { struct LeastSq {
LeastSq() : LeastSq() :
coef(0), coef(0),
rms(0), rms(0),
complexity(benchmark::O_None) {} complexity(benchmark::oNone) {}
double coef; double coef;
double rms; double rms;
@ -41,6 +41,6 @@ struct LeastSq {
}; };
// Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error. // Find the coefficient for the high-order term in the running time, by minimizing the sum of squares of relative error.
LeastSq minimalLeastSq(const std::vector<int>& N, const std::vector<double>& Time, const benchmark::BigO Complexity = benchmark::O_Auto); LeastSq MinimalLeastSq(const std::vector<int>& N, const std::vector<double>& Time, const benchmark::BigO Complexity = benchmark::oAuto);
#endif #endif

View File

@ -95,13 +95,13 @@ void BenchmarkReporter::ComputeBigO(
CpuTime.push_back(run.cpu_accumulated_time/run.iterations); CpuTime.push_back(run.cpu_accumulated_time/run.iterations);
} }
LeastSq resultCpu = minimalLeastSq(N, CpuTime, reports[0].complexity); LeastSq resultCpu = MinimalLeastSq(N, CpuTime, reports[0].complexity);
// resultCpu.complexity is passed as parameter to resultReal because in case // resultCpu.complexity is passed as parameter to resultReal because in case
// reports[0].complexity is O_Auto, the noise on the measured data could make // reports[0].complexity is oAuto, the noise on the measured data could make
// the best fit function of Cpu and Real differ. In order to solve this, we take // the best fit function of Cpu and Real differ. In order to solve this, we take
// the best fitting function for the Cpu, and apply it to Real data. // the best fitting function for the Cpu, and apply it to Real data.
LeastSq resultReal = minimalLeastSq(N, RealTime, resultCpu.complexity); LeastSq resultReal = MinimalLeastSq(N, RealTime, resultCpu.complexity);
std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); std::string benchmark_name = reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/'));
@ -130,17 +130,17 @@ void BenchmarkReporter::ComputeBigO(
std::string BenchmarkReporter::GetBigO(BigO complexity) { std::string BenchmarkReporter::GetBigO(BigO complexity) {
switch (complexity) { switch (complexity) {
case O_N: case oN:
return "* N"; return "* N";
case O_N_Squared: case oNSquared:
return "* N**2"; return "* N**2";
case O_N_Cubed: case oNCubed:
return "* N**3"; return "* N**3";
case O_log_N: case oLogN:
return "* lgN"; return "* lgN";
case O_N_log_N: case oNLogN:
return "* NlgN"; return "* NlgN";
case O_1: case o1:
return "* 1"; return "* 1";
default: default:
return ""; return "";

View File

@ -27,7 +27,7 @@ void BM_Complexity_O1(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
} }
BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::O_1); BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1);
static void BM_Complexity_O_N(benchmark::State& state) { static void BM_Complexity_O_N(benchmark::State& state) {
auto v = ConstructRandomVector(state.range_x()); auto v = ConstructRandomVector(state.range_x());
@ -36,8 +36,8 @@ static void BM_Complexity_O_N(benchmark::State& state) {
benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector)); benchmark::DoNotOptimize(std::find(v.begin(), v.end(), itemNotInVector));
} }
} }
BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN);
BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto);
static void BM_Complexity_O_N_Squared(benchmark::State& state) { static void BM_Complexity_O_N_Squared(benchmark::State& state) {
std::string s1(state.range_x(), '-'); std::string s1(state.range_x(), '-');
@ -50,7 +50,7 @@ static void BM_Complexity_O_N_Squared(benchmark::State& state) {
} }
} }
} }
BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::O_N_Squared); BENCHMARK(BM_Complexity_O_N_Squared) -> Range(1, 1<<8) -> Complexity(benchmark::oNSquared);
static void BM_Complexity_O_N_Cubed(benchmark::State& state) { static void BM_Complexity_O_N_Cubed(benchmark::State& state) {
std::string s1(state.range_x(), '-'); std::string s1(state.range_x(), '-');
@ -67,7 +67,7 @@ static void BM_Complexity_O_N_Cubed(benchmark::State& state) {
} }
} }
} }
BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::O_N_Cubed); BENCHMARK(BM_Complexity_O_N_Cubed) -> DenseRange(1, 8) -> Complexity(benchmark::oNCubed);
static void BM_Complexity_O_log_N(benchmark::State& state) { static void BM_Complexity_O_log_N(benchmark::State& state) {
auto m = ConstructRandomMap(state.range_x()); auto m = ConstructRandomMap(state.range_x());
@ -77,7 +77,7 @@ static void BM_Complexity_O_log_N(benchmark::State& state) {
} }
} }
BENCHMARK(BM_Complexity_O_log_N) BENCHMARK(BM_Complexity_O_log_N)
-> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_log_N); -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oLogN);
static void BM_Complexity_O_N_log_N(benchmark::State& state) { static void BM_Complexity_O_N_log_N(benchmark::State& state) {
auto v = ConstructRandomVector(state.range_x()); auto v = ConstructRandomVector(state.range_x());
@ -85,15 +85,15 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) {
std::sort(v.begin(), v.end()); std::sort(v.begin(), v.end());
} }
} }
BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_N_log_N); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN);
BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::O_Auto); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oAuto);
// Test benchmark with no range and check no complexity is calculated. // Test benchmark with no range and check no complexity is calculated.
void BM_Extreme_Cases(benchmark::State& state) { void BM_Extreme_Cases(benchmark::State& state) {
while (state.KeepRunning()) { while (state.KeepRunning()) {
} }
} }
BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::O_N_log_N); BENCHMARK(BM_Extreme_Cases) -> Complexity(benchmark::oNLogN);
BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::O_Auto); BENCHMARK(BM_Extreme_Cases) -> Arg(42) -> Complexity(benchmark::oAuto);
BENCHMARK_MAIN() BENCHMARK_MAIN()