diff --git a/README.md b/README.md index 31ab13d5..4acd1ff8 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,71 @@ static void BM_MultiThreaded(benchmark::State& state) { BENCHMARK(BM_MultiThreaded)->Threads(2); ``` + +Output Formats +-------------- +The library supports multiple output formats. Use the +`--benchmark_format=` flag to set the format type. `tabular` is +the default format. + +The Tabular format is intended to be a human readable +format. By default the format generates color output. Example tabular output +looks like: +``` +Run on (40 X 2801 MHz CPUs) +2015/03/17-18:35:54 +Build Type: DEBUG +Benchmark Time(ns) CPU(ns) Iterations +---------------------------------------------------------------------- +BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s +BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s +BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s +``` + +The JSON format outputs human readable json split into two top level attributes. +The `context` attribute contains information about the run in general, including +information about the CPU and the date. +The `benchmarks` attribute contains a list of ever benchmark run. Example json +output looks like: +``` +{ + "context": { + "date": "2015/03/17-18:40:25", + "num_cpus": 40, + "mhz_per_cpu": 2801, + "cpu_scaling_enabled": false, + "build_type": "debug" + }, + "benchmarks": [ + { + "name": "BM_SetInsert/1024/1", + "iterations": 94877, + "real_time": 29275, + "cpu_time": 29836, + "bytes_per_second": 134066, + "items_per_second": 33516 + }, + { + "name": "BM_SetInsert/1024/8", + "iterations": 21609, + "real_time": 32317, + "cpu_time": 32429, + "bytes_per_second": 986770, + "items_per_second": 246693 + }, + { + "name": "BM_SetInsert/1024/10", + "iterations": 21393, + "real_time": 32724, + "cpu_time": 33355, + "bytes_per_second": 1199226, + "items_per_second": 299807 + } + ] +} +``` + + Linking against the library --------------------------- When using gcc, it is necessary to link against pthread to avoid runtime exceptions. This is due to how gcc implements std::thread. See [issue #67](https://github.com/google/benchmark/issues/67) for more details. diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index b66854c0..6aebf0c0 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -80,6 +80,8 @@ class BenchmarkReporter { virtual void Finalize(); virtual ~BenchmarkReporter(); +protected: + static void ComputeStats(std::vector const& reports, Run* mean, Run* stddev); }; // Simple reporter that outputs benchmark data to the console. This is the @@ -88,11 +90,24 @@ class ConsoleReporter : public BenchmarkReporter { public: virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - private: +protected: virtual void PrintRunData(const Run& report); size_t name_field_width_; }; +class JSONReporter : public BenchmarkReporter { +public: + JSONReporter() : first_report_(true) {} + virtual bool ReportContext(const Context& context); + virtual void ReportRuns(const std::vector& reports); + virtual void Finalize(); + +private: + void PrintRunData(const Run& report); + + bool first_report_; +}; + } // end namespace benchmark #endif // BENCHMARK_REPORTER_H_ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9b26f0a3..17fb42ad 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,8 +3,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src) # Define the source files set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" "log.cc" - "reporter.cc" "sleep.cc" "string_util.cc" "sysinfo.cc" - "walltime.cc") + "json_reporter.cc" "reporter.cc" "sleep.cc" "string_util.cc" + "sysinfo.cc" "walltime.cc") # Determine the correct regular expression engine to use if(HAVE_STD_REGEX) set(RE_FILES "re_std.cc") diff --git a/src/benchmark.cc b/src/benchmark.cc index 69562fff..36d1d841 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -60,6 +60,10 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); +DEFINE_string(benchmark_format, "tabular", + "The format to use for console output. Valid values are " + "'tabular' or 'json'."); + DEFINE_bool(color_print, true, "Enables colorized logging."); DEFINE_int32(v, 0, "The level of verbose logging to output"); @@ -807,19 +811,35 @@ void RunMatchingBenchmarks(const std::string& spec, } } +std::unique_ptr GetDefaultReporter() { + typedef std::unique_ptr PtrType; + if (FLAGS_benchmark_format == "tabular") { + return PtrType(new ConsoleReporter); + } + else if (FLAGS_benchmark_format == "json") { + return PtrType(new JSONReporter); + } else { + std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; + std::exit(1); + } +} + } // end namespace internal void RunSpecifiedBenchmarks() { RunSpecifiedBenchmarks(nullptr); } -void RunSpecifiedBenchmarks(BenchmarkReporter* provided_reporter) { +void RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks - ConsoleReporter default_reporter; - BenchmarkReporter* reporter = provided_reporter ? provided_reporter - : &default_reporter; + + std::unique_ptr default_reporter; + if (!reporter) { + default_reporter = internal::GetDefaultReporter(); + reporter = default_reporter.get(); + } internal::RunMatchingBenchmarks(spec, reporter); reporter->Finalize(); } @@ -833,6 +853,7 @@ void PrintUsageAndExit() { " [--benchmark_iterations=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_format=]\n" " [--color_print={true|false}]\n" " [--v=]\n"); exit(0); @@ -850,6 +871,8 @@ void ParseCommandLineFlags(int* argc, const char** argv) { &FLAGS_benchmark_min_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseStringFlag(argv[i], "benchmark_format", + &FLAGS_benchmark_format) || ParseBoolFlag(argv[i], "color_print", &FLAGS_color_print) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { @@ -861,6 +884,10 @@ void ParseCommandLineFlags(int* argc, const char** argv) { PrintUsageAndExit(); } } + if (FLAGS_benchmark_format != "tabular" && + FLAGS_benchmark_format != "json") { + PrintUsageAndExit(); + } } } // end namespace internal diff --git a/src/json_reporter.cc b/src/json_reporter.cc new file mode 100644 index 00000000..2ce5cd22 --- /dev/null +++ b/src/json_reporter.cc @@ -0,0 +1,168 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/reporter.h" + +#include +#include +#include +#include + +#include "string_util.h" +#include "walltime.h" + +namespace benchmark { + +namespace { + +std::string FormatKV(std::string const& key, std::string const& value) { + return StringPrintF("\"%s\": \"%s\"", key.c_str(), value.c_str()); +} + +std::string FormatKV(std::string const& key, const char* value) { + return StringPrintF("\"%s\": \"%s\"", key.c_str(), value); +} + +std::string FormatKV(std::string const& key, bool value) { + return StringPrintF("\"%s\": %s", key.c_str(), value ? "true" : "false"); +} + +std::string FormatKV(std::string const& key, int64_t value) { + std::stringstream ss; + ss << '"' << key << "\": " << value; + return ss.str(); +} + +std::string FormatKV(std::string const& key, std::size_t value) { + std::stringstream ss; + ss << '"' << key << "\": " << value; + return ss.str(); +} + +int64_t RoundDouble(double v) { + return static_cast(v + 0.5); +} + +} // end namespace + +bool JSONReporter::ReportContext(const Context& context) { + std::ostream& out = std::cout; + + out << "{\n"; + std::string inner_indent(2, ' '); + + // Open context block and print context information. + out << inner_indent << "\"context\": {\n"; + std::string indent(4, ' '); + int remainder_us; + std::string walltime_value = walltime::Print( + walltime::Now(), "%Y/%m/%d-%H:%M:%S", + true, // use local timezone + &remainder_us); + out << indent << FormatKV("date", walltime_value) << ",\n"; + + out << indent + << FormatKV("num_cpus", static_cast(context.num_cpus)) + << ",\n"; + out << indent + << FormatKV("mhz_per_cpu", RoundDouble(context.mhz_per_cpu)) + << ",\n"; + out << indent + << FormatKV("cpu_scaling_enabled", context.cpu_scaling_enabled) + << ",\n"; + +#if defined(NDEBUG) + const char build_type[] = "release"; +#else + const char build_type[] = "debug"; +#endif + out << indent << FormatKV("build_type", build_type) << "\n"; + // Close context block and open the list of benchmarks. + out << inner_indent << "},\n"; + out << inner_indent << "\"benchmarks\": [\n"; + return true; +} + +void JSONReporter::ReportRuns(std::vector const& reports) { + if (reports.empty()) { + return; + } + std::string indent(4, ' '); + std::ostream& out = std::cout; + if (!first_report_) { + out << ",\n"; + } + first_report_ = false; + std::vector reports_cp = reports; + if (reports.size() >= 2) { + Run mean_data; + Run stddev_data; + BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); + reports_cp.push_back(mean_data); + reports_cp.push_back(stddev_data); + } + for (auto it = reports_cp.begin(); it != reports_cp.end(); ++it) { + out << indent << "{\n"; + PrintRunData(*it); + out << indent << '}'; + auto it_cp = it; + if (++it_cp != reports_cp.end()) { + out << ','; + } + } +} + +void JSONReporter::Finalize() { + // Close the list of benchmarks and the top level object. + std::cout << "\n ]\n}\n"; +} + +void JSONReporter::PrintRunData(Run const& run) { + double const multiplier = 1e9; // nano second multiplier + double cpu_time = run.cpu_accumulated_time * multiplier; + double real_time = run.real_accumulated_time * multiplier; + if (run.iterations != 0) { + real_time = real_time / static_cast(run.iterations); + cpu_time = cpu_time / static_cast(run.iterations); + } + + std::string indent(6, ' '); + std::ostream& out = std::cout; + out << indent + << FormatKV("name", run.benchmark_name) + << ",\n"; + out << indent + << FormatKV("iterations", run.iterations) + << ",\n"; + out << indent + << FormatKV("real_time", RoundDouble(real_time)) + << ",\n"; + out << indent + << FormatKV("cpu_time", RoundDouble(cpu_time)); + if (run.bytes_per_second > 0.0) { + out << ",\n" << indent + << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); + } + if (run.items_per_second > 0.0) { + out << ",\n" << indent + << FormatKV("items_per_second", RoundDouble(run.items_per_second)); + } + if (!run.report_label.empty()) { + out << ",\n" << indent + << FormatKV("label", run.report_label); + } + out << '\n'; +} + +} // end namespace benchmark \ No newline at end of file diff --git a/src/reporter.cc b/src/reporter.cc index 27258c22..233fd9e3 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -26,11 +27,10 @@ #include "walltime.h" namespace benchmark { -namespace { -void ComputeStats(const std::vector& reports, - BenchmarkReporter::Run* mean_data, - BenchmarkReporter::Run* stddev_data) { +void BenchmarkReporter::ComputeStats( + const std::vector& reports, + Run* mean_data, Run* stddev_data) { CHECK(reports.size() >= 2) << "Cannot compute stats for less than 2 reports"; // Accumulators. Stat1_d real_accumulated_time_stat; @@ -42,7 +42,7 @@ void ComputeStats(const std::vector& reports, std::size_t const run_iterations = reports.front().iterations; // Populate the accumulators. - for (BenchmarkReporter::Run const& run : reports) { + for (Run const& run : reports) { CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); CHECK_EQ(run_iterations, run.iterations); real_accumulated_time_stat += @@ -83,8 +83,6 @@ void ComputeStats(const std::vector& reports, stddev_data->items_per_second = items_per_second_stat.StdDev(); } -} // end namespace - void BenchmarkReporter::Finalize() { } @@ -145,7 +143,7 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { Run mean_data; Run stddev_data; - ComputeStats(reports, &mean_data, &stddev_data); + BenchmarkReporter::ComputeStats(reports, &mean_data, &stddev_data); // Output using PrintRun. PrintRunData(mean_data);