benchmark/src/benchmark.cc

630 lines
22 KiB
C++
Raw Normal View History

// Copyright 2015 Google Inc. All rights reserved.
2014-01-10 04:16:51 +08:00
//
2014-01-10 00:01:34 +08:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
2014-01-10 04:16:51 +08:00
//
2014-01-10 00:01:34 +08:00
// http://www.apache.org/licenses/LICENSE-2.0
2014-01-10 04:16:51 +08:00
//
2014-01-10 00:01:34 +08:00
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "benchmark/benchmark.h"
#include "benchmark_api_internal.h"
#include "benchmark_runner.h"
#include "internal_macros.h"
#ifndef BENCHMARK_OS_WINDOWS
#ifndef BENCHMARK_OS_FUCHSIA
#include <sys/resource.h>
#endif
#include <sys/time.h>
#include <unistd.h>
#endif
#include <algorithm>
#include <atomic>
2014-08-04 18:38:37 +08:00
#include <condition_variable>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <limits>
#include <map>
#include <memory>
#include <random>
#include <string>
2014-08-04 18:38:37 +08:00
#include <thread>
#include <utility>
#include "check.h"
#include "colorprint.h"
#include "commandlineflags.h"
#include "complexity.h"
Add user-defined counters. (#262) * Added user counters, and move use of bytes_processed and items_processed to user counter logic. Each counter is a string-value pair. The counters were made available through the State class. Two helper virtual methods were added to the Fixture class to allow convenient initialization and termination of the counters: InitState() and TerminateState(). The reporting of the counters is buggy and is still a work in progress, to be completed in the next commits. * fix bad removal of BenchmarkCounters code during the merge * add myself to AUTHORS/CONTRIBUTORS * fix printing to std::cout in csv_reporter * bytes_per_second and items_per_second are now in the UserCounters class * add user counters to json reporter * moving bytes_per_second and items_per_second to their old state * console reporter dealing ok with user counters. * update unit tests for user counters * CSVReporter now prints user counters too. * cleanup user counters * reverted changes to cmake files which should have gone into later commits * fixture_test: fix gcc 4.6 compilation * remove ctor with default argument see https://github.com/google/benchmark/pull/262#discussion_r72298055 * use (auto-defined) BENCHMARK_HAS_CXX11 instead of BENCHMARK_INITLIST. https://github.com/google/benchmark/pull/262#discussion_r72298310 * leanify counters API Discussions: API complexity: https://github.com/google/benchmark/pull/262#discussion_r72298731 remove std::string dependency (WIP): https://github.com/google/benchmark/pull/262#discussion_r72298142 spacing & alignment: https://github.com/google/benchmark/pull/262#discussion_r72298422 * remove std::string dependency on public API - changed counter name storage to char* * Counter ctor: use overloads instead of default arguments discussion: https://github.com/google/benchmark/pull/262#discussion_r72298055 * Use raw pointers to remove dependency on std::vector from public API . For more info, see discussion at https://github.com/google/benchmark/pull/262#discussion_r72319678 . * Move counter implementation from benchmark.cc to counter.cc. See discussion: https://github.com/google/benchmark/pull/262#discussion_r72298980 . * Remove unused (commented-out) code. * Moved thread counters to ThreadStats. * Counters: fixed copy and move constructors. * Counter: use an inplace buffer for small names. * benchmark_test: move counters test out of CXX11 preprocessor conditional. * Counter: fix VS2013 compilation error in char[] initialization. * Fix typo. * Expose counters from State. See discussion: https://github.com/google/benchmark/pull/262#issuecomment-237156951 * Changed counters interface to map-like. * Fix printing of user counters in ConsoleReporter. * Applied clang-format to counter.cc and console_reporter.cc. Command was `clang-format -style=Google -i counter.cc console_reporter.cc` I also applied to all other files, but the changes were very far-reaching so I rolled those back. * Rename Counter::Flags_e to Counter::Flags * Fix use of reserved names in Counter and BenchmarkCounters. * Counter: Fix move ctor bug + change order of members. * Fixture: remove tentative methods InitState() and TerminateState(). * Update fixture_test to the new Fixture interface. * BenchmarkCounters: fixed a bug in the move ctor. Remove call to CHECK_LT(). CHECK_LT() was making the size_t lookup take ~double the time of a string lookup! * BenchmarkCounters: add option to not print zero counters (defaults to false). * Add test to compare counter storage and access with std::map. * README: clarify cost of counter access modes. * move counter access test to an own test. * BenchmarkCounters: add move Insert() * Counters access test: add accelerated lookup by name. * Fix old range syntax. * Fix missing include of cstdio * Fix Visual Studio warning * VS2013 and lower: fix use of snprintf() * VS2013: fix use of char[] as a member of std::pair<>. * change counter storage to std::map * Remove skipZeroCounters logic * Fix VS compilation error. * Implemented request changes to PR #262. * PR #262: More requested changes. * README: cleanup counter text. * PR #262: remove clang-format changes for preexisting code * Complexity+Counters: fix counter flags which were being ignored. * Document all Counter::Flag members * fixed loss of counter values * ConsoleReporter: remove tabular printing of user counters. * ConsoleReporter: header printing should not be contingent on user counter names. * Minor white space and alignment fixes. * cxx03_test + counters: reuse the BM_empty() function. * user counters: add note to README on how counters are gathered across threads
2017-03-02 08:23:42 +08:00
#include "counter.h"
#include "internal_macros.h"
#include "log.h"
#include "mutex.h"
#include "perf_counters.h"
#include "re.h"
#include "statistics.h"
#include "string_util.h"
#include "thread_manager.h"
#include "thread_timer.h"
namespace benchmark {
// Print a list of benchmarks. This option overrides all other options.
2021-06-25 01:21:59 +08:00
BM_DEFINE_bool(benchmark_list_tests, false);
// A regular expression that specifies the set of benchmarks to execute. If
// this flag is empty, or if this flag is the string \"all\", all benchmarks
// linked into the binary are run.
BM_DEFINE_string(benchmark_filter, "");
// Minimum number of seconds we should run benchmark before results are
// considered significant. For cpu-time based tests, this is the lower bound
// on the total cpu time used by all threads that make up the test. For
// real-time based tests, this is the lower bound on the elapsed time of the
// benchmark execution, regardless of number of threads.
2021-06-25 01:21:59 +08:00
BM_DEFINE_double(benchmark_min_time, 0.5);
// The number of runs of each benchmark. If greater than 1, the mean and
// standard deviation of the runs will be reported.
2021-06-25 01:21:59 +08:00
BM_DEFINE_int32(benchmark_repetitions, 1);
// If set, enable random interleaving of repetitions of all benchmarks.
// See http://github.com/google/benchmark/issues/1051 for details.
2021-06-25 01:21:59 +08:00
BM_DEFINE_bool(benchmark_enable_random_interleaving, false);
// Report the result of each benchmark repetitions. When 'true' is specified
// only the mean, standard deviation, and other statistics are reported for
// repeated benchmarks. Affects all reporters.
2021-06-25 01:21:59 +08:00
BM_DEFINE_bool(benchmark_report_aggregates_only, false);
// Display the result of each benchmark repetitions. When 'true' is specified
// only the mean, standard deviation, and other statistics are displayed for
// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
// the display reporter, but *NOT* file reporter, which will still contain
// all the output.
2021-06-25 01:21:59 +08:00
BM_DEFINE_bool(benchmark_display_aggregates_only, false);
// The format to use for console output.
// Valid values are 'console', 'json', or 'csv'.
2021-06-25 01:21:59 +08:00
BM_DEFINE_string(benchmark_format, "console");
// The format to use for file output.
// Valid values are 'console', 'json', or 'csv'.
2021-06-25 01:21:59 +08:00
BM_DEFINE_string(benchmark_out_format, "json");
// The file to write additional output to.
2021-06-25 01:21:59 +08:00
BM_DEFINE_string(benchmark_out, "");
// Whether to use colors in the output. Valid values:
// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
// the output is being sent to a terminal and the TERM environment variable is
// set to a terminal type that supports colors.
2021-06-25 01:21:59 +08:00
BM_DEFINE_string(benchmark_color, "auto");
// Whether to use tabular format when printing user counters to the console.
// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
2021-06-25 01:21:59 +08:00
BM_DEFINE_bool(benchmark_counters_tabular, false);
// List of additional perf counters to collect, in libpfm format. For more
// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html
2021-06-25 01:21:59 +08:00
BM_DEFINE_string(benchmark_perf_counters, "");
// Extra context to include in the output formatted as comma-separated key-value
// pairs. Kept internal as it's only used for parsing from env/command line.
2021-06-25 01:21:59 +08:00
BM_DEFINE_kvpairs(benchmark_context, {});
// The level of verbose logging to output
2021-06-25 01:21:59 +08:00
BM_DEFINE_int32(v, 0);
namespace internal {
std::map<std::string, std::string>* global_context = nullptr;
// FIXME: wouldn't LTO mess this up?
void UseCharPointer(char const volatile*) {}
} // namespace internal
Iteration counts should be `uint64_t` globally. (#817) This is a shameless rip-off of https://github.com/google/benchmark/pull/646 I did promise to look into why that proposed PR was producing so much worse assembly, and so i finally did. The reason is - that diff changes `size_t` (unsigned) to `int64_t` (signed). There is this nice little `assert`: https://github.com/google/benchmark/blob/7a1c37028359ca9d386d719a6ad527743cf1b753/include/benchmark/benchmark.h#L744 It ensures that we didn't magically decide to advance our iterator when we should have finished benchmarking. When `cached_` was unsigned, the `assert` was `cached_ UGT 0`. But we only ever get to that `assert` if `cached_ NE 0`, and naturally if `cached_` is not `0`, then it is bigger than `0`, so the `assert` is tautological, and gets folded away. But now that `cached_` became signed, the assert became `cached_ SGT 0`. And we still only know that `cached_ NE 0`, so the assert can't be optimized out, or at least it doesn't currently. Regardless of whether or not that is a bug in itself, that particular diff would have regressed the normal 64-bit systems, by halving the maximal iteration space (since we go from unsigned counter to signed one, of the same bit-width), which seems like a bug. And just so it happens, fixing *this* bug, fixes the other bug. This produces fully (bit-by-bit) identical state_assembly_test.s The filecheck change is actually needed regardless of this patch, else this test does not pass for me even without this diff.
2019-05-13 17:33:11 +08:00
State::State(IterationCount max_iters, const std::vector<int64_t>& ranges,
int thread_i, int n_threads, internal::ThreadTimer* timer,
internal::ThreadManager* manager,
internal::PerfCountersMeasurement* perf_counters_measurement)
: total_iterations_(0),
batch_leftover_(0),
max_iterations(max_iters),
started_(false),
finished_(false),
error_occurred_(false),
range_(ranges),
2016-05-25 02:06:54 +08:00
complexity_n_(0),
Introduce accessors for currently public data members (threads and thread_index) (#1208) * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate the direct access to these fields. Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate the direct access to these fields. Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else.
2021-08-23 16:06:57 +08:00
thread_index_(thread_i),
threads_(n_threads),
timer_(timer),
manager_(manager),
perf_counters_measurement_(perf_counters_measurement) {
2021-06-25 01:21:59 +08:00
BM_CHECK(max_iterations != 0) << "At least one iteration must be run";
Introduce accessors for currently public data members (threads and thread_index) (#1208) * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate the direct access to these fields. Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate the direct access to these fields. Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else. * [benchmark] Introduce accessors for currently public data members `threads` and `thread_index` Also deprecate direct access to `.thread_index` and make threads a private field Motivations: Our internal library provides accessors for those fields because the styleguide disalows accessing classes' data members directly (even if they're const). There has been a discussion to simply move internal library to make its fields public similarly to the OSS version here, however, the concern is that these kinds of direct access would prevent many types of future design changes (eg how/whether the values would be stored in the data member) I think the concensus in the end is that we'd change the external library for this case. AFAIK, there are three important third_party users that we'd need to migrate: tcmalloc, abseil and tensorflow. Please let me know if I'm missing anyone else.
2021-08-23 16:06:57 +08:00
BM_CHECK_LT(thread_index_, threads_)
2021-06-25 01:21:59 +08:00
<< "thread_index must be less than threads";
// Note: The use of offsetof below is technically undefined until C++17
// because State is not a standard layout type. However, all compilers
// currently provide well-defined behavior as an extension (which is
// demonstrated since constexpr evaluation must diagnose all undefined
// behavior). However, GCC and Clang also warn about this use of offsetof,
// which must be suppressed.
#if defined(__INTEL_COMPILER)
#pragma warning push
#pragma warning(disable : 1875)
#elif defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winvalid-offsetof"
#endif
// Offset tests to ensure commonly accessed data is on the first cache line.
const int cache_line_size = 64;
static_assert(offsetof(State, error_occurred_) <=
(cache_line_size - sizeof(error_occurred_)),
"");
#if defined(__INTEL_COMPILER)
#pragma warning pop
#elif defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
}
void State::PauseTiming() {
// Add in time accumulated so far
2021-06-25 01:21:59 +08:00
BM_CHECK(started_ && !finished_ && !error_occurred_);
timer_->StopTimer();
if (perf_counters_measurement_) {
std::vector<std::pair<std::string, double>> measurements;
if (!perf_counters_measurement_->Stop(measurements)) {
BM_CHECK(false) << "Perf counters read the value failed.";
}
for (const auto& name_and_measurement : measurements) {
auto name = name_and_measurement.first;
auto measurement = name_and_measurement.second;
2021-06-25 01:21:59 +08:00
BM_CHECK_EQ(counters[name], 0.0);
counters[name] = Counter(measurement, Counter::kAvgIterations);
}
}
}
void State::ResumeTiming() {
2021-06-25 01:21:59 +08:00
BM_CHECK(started_ && !finished_ && !error_occurred_);
timer_->StartTimer();
if (perf_counters_measurement_) {
perf_counters_measurement_->Start();
}
}
2016-05-24 09:24:56 +08:00
void State::SkipWithError(const char* msg) {
2021-06-25 01:21:59 +08:00
BM_CHECK(msg);
2016-05-24 09:24:56 +08:00
error_occurred_ = true;
{
MutexLock l(manager_->GetBenchmarkMutex());
if (manager_->results.has_error_ == false) {
manager_->results.error_message_ = msg;
manager_->results.has_error_ = true;
}
}
total_iterations_ = 0;
if (timer_->running()) timer_->StopTimer();
2016-05-24 09:24:56 +08:00
}
void State::SetIterationTime(double seconds) {
timer_->SetIterationTime(seconds);
2016-04-30 21:23:58 +08:00
}
void State::SetLabel(const char* label) {
MutexLock l(manager_->GetBenchmarkMutex());
manager_->results.report_label_ = label;
}
void State::StartKeepRunning() {
2021-06-25 01:21:59 +08:00
BM_CHECK(!started_ && !finished_);
started_ = true;
total_iterations_ = error_occurred_ ? 0 : max_iterations;
manager_->StartStopBarrier();
if (!error_occurred_) ResumeTiming();
}
void State::FinishKeepRunning() {
2021-06-25 01:21:59 +08:00
BM_CHECK(started_ && (!finished_ || error_occurred_));
if (!error_occurred_) {
PauseTiming();
}
// Total iterations has now wrapped around past 0. Fix this.
total_iterations_ = 0;
finished_ = true;
manager_->StartStopBarrier();
}
namespace internal {
2015-03-31 11:32:37 +08:00
namespace {
// Flushes streams after invoking reporter methods that write to them. This
// ensures users get timely updates even when streams are not line-buffered.
void FlushStreams(BenchmarkReporter* reporter) {
if (!reporter) return;
std::flush(reporter->GetOutputStream());
std::flush(reporter->GetErrorStream());
}
// Reports in both display and file reporters.
void Report(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter, const RunResults& run_results) {
auto report_one = [](BenchmarkReporter* reporter, bool aggregates_only,
const RunResults& results) {
assert(reporter);
// If there are no aggregates, do output non-aggregates.
aggregates_only &= !results.aggregates_only.empty();
if (!aggregates_only) reporter->ReportRuns(results.non_aggregates);
if (!results.aggregates_only.empty())
reporter->ReportRuns(results.aggregates_only);
};
report_one(display_reporter, run_results.display_report_aggregates_only,
run_results);
if (file_reporter)
report_one(file_reporter, run_results.file_report_aggregates_only,
run_results);
FlushStreams(display_reporter);
FlushStreams(file_reporter);
}
void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter) {
// Note the file_reporter can be null.
2021-06-25 01:21:59 +08:00
BM_CHECK(display_reporter != nullptr);
// Determine the width of the name field using a minimum width of 10.
bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
size_t name_field_width = 10;
Drop Stat1, refactor statistics to be user-providable, add median. (#428) * Drop Stat1, refactor statistics to be user-providable, add median. My main goal was to add median statistic. Since Stat1 calculated the stats incrementally, and did not store the values themselves, it is was not possible. Thus, i have replaced Stat1 with simple std::vector<double>, containing all the values. Then, i have refactored current mean/stdev to be a function that is provided with values vector, and returns the statistic. While there, it seemed to make sense to deduplicate the code by storing all the statistics functions in a map, and then simply iterate over it. And the interface to add new statistics is intentionally exposed, so they may be added easily. The notable change is that Iterations are no longer displayed as 0 for stdev. Is could be changed, but i'm not sure how to nicely fit that into the API. Similarly, this dance about sometimes (for some fields, for some statistics) dividing by run.iterations, and then multiplying the calculated stastic back is also dropped, and if you do the math, i fail to see why it was needed there in the first place. Since that was the only use of stat.h, it is removed. * complexity.h: attempt to fix MSVC build * Update README.md * Store statistics to compute in a vector, ensures ordering. * Add a bit more tests for repetitions. * Partially address review notes. * Fix gcc build: drop extra ';' clang, why didn't you warn me? * Address review comments. * double() -> 0.0 * early return
2017-08-24 07:44:29 +08:00
size_t stat_field_width = 0;
for (const BenchmarkInstance& benchmark : benchmarks) {
name_field_width =
std::max<size_t>(name_field_width, benchmark.name().str().size());
might_have_aggregates |= benchmark.repetitions() > 1;
Drop Stat1, refactor statistics to be user-providable, add median. (#428) * Drop Stat1, refactor statistics to be user-providable, add median. My main goal was to add median statistic. Since Stat1 calculated the stats incrementally, and did not store the values themselves, it is was not possible. Thus, i have replaced Stat1 with simple std::vector<double>, containing all the values. Then, i have refactored current mean/stdev to be a function that is provided with values vector, and returns the statistic. While there, it seemed to make sense to deduplicate the code by storing all the statistics functions in a map, and then simply iterate over it. And the interface to add new statistics is intentionally exposed, so they may be added easily. The notable change is that Iterations are no longer displayed as 0 for stdev. Is could be changed, but i'm not sure how to nicely fit that into the API. Similarly, this dance about sometimes (for some fields, for some statistics) dividing by run.iterations, and then multiplying the calculated stastic back is also dropped, and if you do the math, i fail to see why it was needed there in the first place. Since that was the only use of stat.h, it is removed. * complexity.h: attempt to fix MSVC build * Update README.md * Store statistics to compute in a vector, ensures ordering. * Add a bit more tests for repetitions. * Partially address review notes. * Fix gcc build: drop extra ';' clang, why didn't you warn me? * Address review comments. * double() -> 0.0 * early return
2017-08-24 07:44:29 +08:00
for (const auto& Stat : benchmark.statistics())
Drop Stat1, refactor statistics to be user-providable, add median. (#428) * Drop Stat1, refactor statistics to be user-providable, add median. My main goal was to add median statistic. Since Stat1 calculated the stats incrementally, and did not store the values themselves, it is was not possible. Thus, i have replaced Stat1 with simple std::vector<double>, containing all the values. Then, i have refactored current mean/stdev to be a function that is provided with values vector, and returns the statistic. While there, it seemed to make sense to deduplicate the code by storing all the statistics functions in a map, and then simply iterate over it. And the interface to add new statistics is intentionally exposed, so they may be added easily. The notable change is that Iterations are no longer displayed as 0 for stdev. Is could be changed, but i'm not sure how to nicely fit that into the API. Similarly, this dance about sometimes (for some fields, for some statistics) dividing by run.iterations, and then multiplying the calculated stastic back is also dropped, and if you do the math, i fail to see why it was needed there in the first place. Since that was the only use of stat.h, it is removed. * complexity.h: attempt to fix MSVC build * Update README.md * Store statistics to compute in a vector, ensures ordering. * Add a bit more tests for repetitions. * Partially address review notes. * Fix gcc build: drop extra ';' clang, why didn't you warn me? * Address review comments. * double() -> 0.0 * early return
2017-08-24 07:44:29 +08:00
stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size());
}
if (might_have_aggregates) name_field_width += 1 + stat_field_width;
// Print header here
2014-01-08 08:33:40 +08:00
BenchmarkReporter::Context context;
context.name_field_width = name_field_width;
// Keep track of running times of all instances of each benchmark family.
std::map<int /*family_index*/, BenchmarkReporter::PerFamilyRunReports>
per_family_reports;
2016-05-19 03:25:00 +08:00
if (display_reporter->ReportContext(context) &&
(!file_reporter || file_reporter->ReportContext(context))) {
FlushStreams(display_reporter);
FlushStreams(file_reporter);
size_t num_repetitions_total = 0;
std::vector<internal::BenchmarkRunner> runners;
runners.reserve(benchmarks.size());
for (const BenchmarkInstance& benchmark : benchmarks) {
BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr;
if (benchmark.complexity() != oNone)
reports_for_family = &per_family_reports[benchmark.family_index()];
runners.emplace_back(benchmark, reports_for_family);
int num_repeats_of_this_instance = runners.back().GetNumRepeats();
num_repetitions_total += num_repeats_of_this_instance;
if (reports_for_family)
reports_for_family->num_runs_total += num_repeats_of_this_instance;
}
assert(runners.size() == benchmarks.size() && "Unexpected runner count.");
std::vector<size_t> repetition_indices;
repetition_indices.reserve(num_repetitions_total);
for (size_t runner_index = 0, num_runners = runners.size();
runner_index != num_runners; ++runner_index) {
const internal::BenchmarkRunner& runner = runners[runner_index];
std::fill_n(std::back_inserter(repetition_indices),
runner.GetNumRepeats(), runner_index);
}
assert(repetition_indices.size() == num_repetitions_total &&
"Unexpected number of repetition indexes.");
if (FLAGS_benchmark_enable_random_interleaving) {
std::random_device rd;
std::mt19937 g(rd());
std::shuffle(repetition_indices.begin(), repetition_indices.end(), g);
}
for (size_t repetition_index : repetition_indices) {
internal::BenchmarkRunner& runner = runners[repetition_index];
runner.DoOneRepetition();
if (runner.HasRepeatsRemaining()) continue;
// FIXME: report each repetition separately, not all of them in bulk.
RunResults run_results = runner.GetResults();
// Maybe calculate complexity report
if (const auto* reports_for_family = runner.GetReportsForFamily()) {
if (reports_for_family->num_runs_done ==
reports_for_family->num_runs_total) {
auto additional_run_stats = ComputeBigO(reports_for_family->Runs);
run_results.aggregates_only.insert(run_results.aggregates_only.end(),
additional_run_stats.begin(),
additional_run_stats.end());
per_family_reports.erase(
static_cast<int>(reports_for_family->Runs.front().family_index));
}
}
Report(display_reporter, file_reporter, run_results);
}
}
display_reporter->Finalize();
if (file_reporter) file_reporter->Finalize();
FlushStreams(display_reporter);
FlushStreams(file_reporter);
}
// Disable deprecated warnings temporarily because we need to reference
// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
BENCHMARK_DISABLE_DEPRECATED_WARNING
std::unique_ptr<BenchmarkReporter> CreateReporter(
std::string const& name, ConsoleReporter::OutputOptions output_opts) {
2015-03-18 06:18:06 +08:00
typedef std::unique_ptr<BenchmarkReporter> PtrType;
if (name == "console") {
return PtrType(new ConsoleReporter(output_opts));
} else if (name == "json") {
2015-03-18 06:18:06 +08:00
return PtrType(new JSONReporter);
} else if (name == "csv") {
2015-03-28 04:27:15 +08:00
return PtrType(new CSVReporter);
2015-03-18 06:18:06 +08:00
} else {
std::cerr << "Unexpected format: '" << name << "'\n";
2015-03-18 06:18:06 +08:00
std::exit(1);
}
}
BENCHMARK_RESTORE_DEPRECATED_WARNING
} // end namespace
bool IsZero(double n) {
return std::abs(n) < std::numeric_limits<double>::epsilon();
}
ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {
int output_opts = ConsoleReporter::OO_Defaults;
auto is_benchmark_color = [force_no_color]() -> bool {
if (force_no_color) {
return false;
}
if (FLAGS_benchmark_color == "auto") {
return IsColorTerminal();
}
return IsTruthyFlagValue(FLAGS_benchmark_color);
};
if (is_benchmark_color()) {
output_opts |= ConsoleReporter::OO_Color;
} else {
output_opts &= ~ConsoleReporter::OO_Color;
}
if (FLAGS_benchmark_counters_tabular) {
output_opts |= ConsoleReporter::OO_Tabular;
} else {
output_opts &= ~ConsoleReporter::OO_Tabular;
}
return static_cast<ConsoleReporter::OutputOptions>(output_opts);
}
} // end namespace internal
size_t RunSpecifiedBenchmarks() {
return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(std::string spec) {
return RunSpecifiedBenchmarks(nullptr, nullptr, std::move(spec));
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) {
return RunSpecifiedBenchmarks(display_reporter, nullptr,
FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
std::string spec) {
return RunSpecifiedBenchmarks(display_reporter, nullptr, std::move(spec));
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter) {
return RunSpecifiedBenchmarks(display_reporter, file_reporter,
FLAGS_benchmark_filter);
}
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
BenchmarkReporter* file_reporter,
std::string spec) {
if (spec.empty() || spec == "all")
2014-01-10 04:12:11 +08:00
spec = "."; // Regexp that matches all benchmarks
2015-03-18 06:18:06 +08:00
// Setup the reporters
std::ofstream output_file;
std::unique_ptr<BenchmarkReporter> default_display_reporter;
std::unique_ptr<BenchmarkReporter> default_file_reporter;
if (!display_reporter) {
default_display_reporter = internal::CreateReporter(
FLAGS_benchmark_format, internal::GetOutputOptions());
display_reporter = default_display_reporter.get();
}
auto& Out = display_reporter->GetOutputStream();
auto& Err = display_reporter->GetErrorStream();
std::string const& fname = FLAGS_benchmark_out;
2017-07-14 00:33:43 +08:00
if (fname.empty() && file_reporter) {
Err << "A custom file reporter was provided but "
"--benchmark_out=<file> was not specified."
<< std::endl;
std::exit(1);
}
2017-07-14 00:33:43 +08:00
if (!fname.empty()) {
output_file.open(fname);
if (!output_file.is_open()) {
Err << "invalid file name: '" << fname << "'" << std::endl;
std::exit(1);
}
if (!file_reporter) {
default_file_reporter = internal::CreateReporter(
FLAGS_benchmark_out_format, ConsoleReporter::OO_None);
file_reporter = default_file_reporter.get();
}
file_reporter->SetOutputStream(&output_file);
file_reporter->SetErrorStream(&output_file);
}
std::vector<internal::BenchmarkInstance> benchmarks;
if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0;
if (benchmarks.empty()) {
Err << "Failed to match any benchmarks against regex: " << spec << "\n";
return 0;
}
if (FLAGS_benchmark_list_tests) {
for (auto const& benchmark : benchmarks)
Out << benchmark.name().str() << "\n";
} else {
internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);
2015-03-18 06:18:06 +08:00
}
return benchmarks.size();
}
std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; }
void RegisterMemoryManager(MemoryManager* manager) {
internal::memory_manager = manager;
}
void AddCustomContext(const std::string& key, const std::string& value) {
if (internal::global_context == nullptr) {
internal::global_context = new std::map<std::string, std::string>();
}
if (!internal::global_context->emplace(key, value).second) {
std::cerr << "Failed to add custom context \"" << key << "\" as it already "
<< "exists with value \"" << value << "\"\n";
}
}
namespace internal {
void PrintUsageAndExit() {
fprintf(stdout,
"benchmark"
2015-03-31 11:32:37 +08:00
" [--benchmark_list_tests={true|false}]\n"
" [--benchmark_filter=<regex>]\n"
" [--benchmark_min_time=<min_time>]\n"
" [--benchmark_repetitions=<num_repetitions>]\n"
" [--benchmark_enable_random_interleaving={true|false}]\n"
" [--benchmark_report_aggregates_only={true|false}]\n"
" [--benchmark_display_aggregates_only={true|false}]\n"
" [--benchmark_format=<console|json|csv>]\n"
" [--benchmark_out=<filename>]\n"
" [--benchmark_out_format=<json|console|csv>]\n"
" [--benchmark_color={auto|true|false}]\n"
" [--benchmark_counters_tabular={true|false}]\n"
" [--benchmark_perf_counters=<counter>,...]\n"
" [--benchmark_context=<key>=<value>,...]\n"
" [--v=<verbosity>]\n");
exit(0);
}
void ParseCommandLineFlags(int* argc, char** argv) {
using namespace benchmark;
BenchmarkReporter::Context::executable_name =
(argc && *argc > 0) ? argv[0] : "unknown";
for (int i = 1; argc && i < *argc; ++i) {
if (ParseBoolFlag(argv[i], "benchmark_list_tests",
2015-03-31 11:32:37 +08:00
&FLAGS_benchmark_list_tests) ||
ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) ||
ParseDoubleFlag(argv[i], "benchmark_min_time",
&FLAGS_benchmark_min_time) ||
ParseInt32Flag(argv[i], "benchmark_repetitions",
&FLAGS_benchmark_repetitions) ||
ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving",
&FLAGS_benchmark_enable_random_interleaving) ||
ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
&FLAGS_benchmark_report_aggregates_only) ||
ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
&FLAGS_benchmark_display_aggregates_only) ||
ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
ParseStringFlag(argv[i], "benchmark_out_format",
&FLAGS_benchmark_out_format) ||
ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) ||
ParseBoolFlag(argv[i], "benchmark_counters_tabular",
&FLAGS_benchmark_counters_tabular) ||
ParseStringFlag(argv[i], "benchmark_perf_counters",
&FLAGS_benchmark_perf_counters) ||
ParseKeyValueFlag(argv[i], "benchmark_context",
&FLAGS_benchmark_context) ||
ParseInt32Flag(argv[i], "v", &FLAGS_v)) {
for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1];
--(*argc);
--i;
} else if (IsFlag(argv[i], "help")) {
PrintUsageAndExit();
}
}
for (auto const* flag :
{&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) {
if (*flag != "console" && *flag != "json" && *flag != "csv") {
PrintUsageAndExit();
}
}
if (FLAGS_benchmark_color.empty()) {
PrintUsageAndExit();
}
for (const auto& kv : FLAGS_benchmark_context) {
AddCustomContext(kv.first, kv.second);
}
}
int InitializeStreams() {
static std::ios_base::Init init;
return 0;
}
} // end namespace internal
void Initialize(int* argc, char** argv) {
2014-01-10 04:12:11 +08:00
internal::ParseCommandLineFlags(argc, argv);
internal::LogLevel() = FLAGS_v;
}
void Shutdown() { delete internal::global_context; }
bool ReportUnrecognizedArguments(int argc, char** argv) {
for (int i = 1; i < argc; ++i) {
fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0],
argv[i]);
}
return argc > 1;
}
} // end namespace benchmark