Added the functionality for a dry run benchmark called through the cli argument --benchmark_dry_run. (#1851)
Some checks are pending
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (false, macos-latest) (push) Waiting to run
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (false, ubuntu-latest) (push) Waiting to run
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (false, windows-latest) (push) Waiting to run
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (true, macos-latest) (push) Waiting to run
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (true, ubuntu-latest) (push) Waiting to run
bazel / bazel.${{ matrix.os }}.${{ matrix.bzlmod && 'bzlmod' || 'no_bzlmod' }} (true, windows-latest) (push) Waiting to run
build-and-test-min-cmake / ${{ matrix.os }}.min-cmake (macos-latest) (push) Waiting to run
build-and-test-min-cmake / ${{ matrix.os }}.min-cmake (ubuntu-latest) (push) Waiting to run
build-and-test-perfcounters / ${{ matrix.os }}.${{ matrix.build_type }} (Debug, ubuntu-20.04) (push) Waiting to run
build-and-test-perfcounters / ${{ matrix.os }}.${{ matrix.build_type }} (Debug, ubuntu-22.04) (push) Waiting to run
build-and-test-perfcounters / ${{ matrix.os }}.${{ matrix.build_type }} (Release, ubuntu-20.04) (push) Waiting to run
build-and-test-perfcounters / ${{ matrix.os }}.${{ matrix.build_type }} (Release, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, shared, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, shared, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, shared, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, static, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, static, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, clang++, static, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, shared, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, shared, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, shared, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, static, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, static, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Debug, g++, static, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, shared, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, shared, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, shared, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, static, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, static, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, clang++, static, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, shared, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, shared, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, shared, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, static, macos-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, static, ubuntu-20.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.compiler }} (Release, g++, static, ubuntu-22.04) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Debug, Visual Studio 16 2019, shared, VS-16-2019, windows-2019) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Debug, Visual Studio 16 2019, static, VS-16-2019, windows-2019) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Debug, Visual Studio 17 2022, shared, VS-17-2022, windows-2022) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Debug, Visual Studio 17 2022, static, VS-17-2022, windows-2022) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Release, Visual Studio 16 2019, shared, VS-16-2019, windows-2019) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Release, Visual Studio 16 2019, static, VS-16-2019, windows-2019) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Release, Visual Studio 17 2022, shared, VS-17-2022, windows-2022) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msvc }} (x64, Release, Visual Studio 17 2022, static, VS-17-2022, windows-2022) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, shared, map[arch:i686 compiler:clang++ family:LLVM msystem:CLANG32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, shared, map[arch:i686 compiler:g++ family:GNU msystem:MINGW32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, shared, map[arch:x86_64 compiler:clang++ family:LLVM msystem:CLANG64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, shared, map[arch:x86_64 compiler:g++ family:GNU msystem:MINGW64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, shared, map[arch:x86_64 compiler:g++ family:GNU msystem:UCRT64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, static, map[arch:i686 compiler:clang++ family:LLVM msystem:CLANG32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, static, map[arch:i686 compiler:g++ family:GNU msystem:MINGW32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, static, map[arch:x86_64 compiler:clang++ family:LLVM msystem:CLANG64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, static, map[arch:x86_64 compiler:g++ family:GNU msystem:MINGW64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Debug, static, map[arch:x86_64 compiler:g++ family:GNU msystem:UCRT64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, shared, map[arch:i686 compiler:clang++ family:LLVM msystem:CLANG32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, shared, map[arch:i686 compiler:g++ family:GNU msystem:MINGW32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, shared, map[arch:x86_64 compiler:clang++ family:LLVM msystem:CLANG64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, shared, map[arch:x86_64 compiler:g++ family:GNU msystem:MINGW64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, shared, map[arch:x86_64 compiler:g++ family:GNU msystem:UCRT64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, static, map[arch:i686 compiler:clang++ family:LLVM msystem:CLANG32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, static, map[arch:i686 compiler:g++ family:GNU msystem:MINGW32], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, static, map[arch:x86_64 compiler:clang++ family:LLVM msystem:CLANG64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, static, map[arch:x86_64 compiler:g++ family:GNU msystem:MINGW64], windows-latest) (push) Waiting to run
build-and-test / ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.lib }}.${{ matrix.msys2.msystem }} (Release, static, map[arch:x86_64 compiler:g++ family:GNU msystem:UCRT64], windows-latest) (push) Waiting to run
clang-format-lint / check-clang-format (push) Waiting to run
clang-tidy / run-clang-tidy (push) Waiting to run
doxygen / Build HTML documentation (push) Waiting to run
python + Bazel pre-commit checks / pre-commit (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (Debug, asan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (Debug, msan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (Debug, tsan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (Debug, ubsan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (RelWithDebInfo, asan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (RelWithDebInfo, msan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (RelWithDebInfo, tsan) (push) Waiting to run
sanitizer / ${{ matrix.sanitizer }}.${{ matrix.build_type }} (RelWithDebInfo, ubsan) (push) Waiting to run
test-bindings / Test GBM Python bindings on ${{ matrix.os }} (macos-latest) (push) Waiting to run
test-bindings / Test GBM Python bindings on ${{ matrix.os }} (ubuntu-latest) (push) Waiting to run
test-bindings / Test GBM Python bindings on ${{ matrix.os }} (windows-latest) (push) Waiting to run

* Added benchmark_dry_run boolean flag to command line options

* Dry run logic to exit early and override iterations, repetitions, min time, min warmup time

* Changeddry run override logic structure and added dry run to context

---------

Co-authored-by: Shaan <shaanmistry03@gmail.com>
Co-authored-by: Shaan Mistry <49106143+Shaan-Mistry@users.noreply.github.com>
This commit is contained in:
dominic 2024-09-12 15:50:52 +01:00 committed by GitHub
parent 08fdf6eb84
commit 72ecc4ea67
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 38 additions and 18 deletions

View File

@ -92,6 +92,11 @@ BM_DEFINE_double(benchmark_min_warmup_time, 0.0);
// standard deviation of the runs will be reported.
BM_DEFINE_int32(benchmark_repetitions, 1);
// If enabled, forces each benchmark to execute exactly one iteration and one
// repetition, bypassing any configured
// MinTime()/MinWarmUpTime()/Iterations()/Repetitions()
BM_DEFINE_bool(benchmark_dry_run, false);
// If set, enable random interleaving of repetitions of all benchmarks.
// See http://github.com/google/benchmark/issues/1051 for details.
BM_DEFINE_bool(benchmark_enable_random_interleaving, false);
@ -717,6 +722,7 @@ void ParseCommandLineFlags(int* argc, char** argv) {
&FLAGS_benchmark_min_warmup_time) ||
ParseInt32Flag(argv[i], "benchmark_repetitions",
&FLAGS_benchmark_repetitions) ||
ParseBoolFlag(argv[i], "benchmark_dry_run", &FLAGS_benchmark_dry_run) ||
ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving",
&FLAGS_benchmark_enable_random_interleaving) ||
ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
@ -755,6 +761,9 @@ void ParseCommandLineFlags(int* argc, char** argv) {
if (FLAGS_benchmark_color.empty()) {
PrintUsageAndExit();
}
if (FLAGS_benchmark_dry_run) {
AddCustomContext("dry_run", "true");
}
for (const auto& kv : FLAGS_benchmark_context) {
AddCustomContext(kv.first, kv.second);
}
@ -783,6 +792,7 @@ void PrintDefaultHelp() {
" [--benchmark_min_time=`<integer>x` OR `<float>s` ]\n"
" [--benchmark_min_warmup_time=<min_warmup_time>]\n"
" [--benchmark_repetitions=<num_repetitions>]\n"
" [--benchmark_dry_run={true|false}]\n"
" [--benchmark_enable_random_interleaving={true|false}]\n"
" [--benchmark_report_aggregates_only={true|false}]\n"
" [--benchmark_display_aggregates_only={true|false}]\n"

View File

@ -58,6 +58,14 @@
namespace benchmark {
BM_DECLARE_bool(benchmark_dry_run);
BM_DECLARE_string(benchmark_min_time);
BM_DECLARE_double(benchmark_min_warmup_time);
BM_DECLARE_int32(benchmark_repetitions);
BM_DECLARE_bool(benchmark_report_aggregates_only);
BM_DECLARE_bool(benchmark_display_aggregates_only);
BM_DECLARE_string(benchmark_perf_counters);
namespace internal {
MemoryManager* memory_manager = nullptr;
@ -228,20 +236,29 @@ BenchmarkRunner::BenchmarkRunner(
: b(b_),
reports_for_family(reports_for_family_),
parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)),
min_time(ComputeMinTime(b_, parsed_benchtime_flag)),
min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0)
? b.min_warmup_time()
: FLAGS_benchmark_min_warmup_time),
warmup_done(!(min_warmup_time > 0.0)),
repeats(b.repetitions() != 0 ? b.repetitions()
: FLAGS_benchmark_repetitions),
min_time(FLAGS_benchmark_dry_run
? 0
: ComputeMinTime(b_, parsed_benchtime_flag)),
min_warmup_time(
FLAGS_benchmark_dry_run
? 0
: ((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0)
? b.min_warmup_time()
: FLAGS_benchmark_min_warmup_time)),
warmup_done(FLAGS_benchmark_dry_run ? true : !(min_warmup_time > 0.0)),
repeats(FLAGS_benchmark_dry_run
? 1
: (b.repetitions() != 0 ? b.repetitions()
: FLAGS_benchmark_repetitions)),
has_explicit_iteration_count(b.iterations() != 0 ||
parsed_benchtime_flag.tag ==
BenchTimeType::ITERS),
pool(static_cast<size_t>(b.threads() - 1)),
iters(has_explicit_iteration_count
? ComputeIters(b_, parsed_benchtime_flag)
: 1),
iters(FLAGS_benchmark_dry_run
? 1
: (has_explicit_iteration_count
? ComputeIters(b_, parsed_benchtime_flag)
: 1)),
perf_counters_measurement_ptr(pcm_) {
run_results.display_report_aggregates_only =
(FLAGS_benchmark_report_aggregates_only ||
@ -339,7 +356,7 @@ bool BenchmarkRunner::ShouldReportIterationResults(
// Determine if this run should be reported;
// Either it has run for a sufficient amount of time
// or because an error was reported.
return i.results.skipped_ ||
return i.results.skipped_ || FLAGS_benchmark_dry_run ||
i.iters >= kMaxIterations || // Too many iterations already.
i.seconds >=
GetMinTimeToApply() || // The elapsed time is large enough.

View File

@ -25,13 +25,6 @@
namespace benchmark {
BM_DECLARE_string(benchmark_min_time);
BM_DECLARE_double(benchmark_min_warmup_time);
BM_DECLARE_int32(benchmark_repetitions);
BM_DECLARE_bool(benchmark_report_aggregates_only);
BM_DECLARE_bool(benchmark_display_aggregates_only);
BM_DECLARE_string(benchmark_perf_counters);
namespace internal {
extern MemoryManager* memory_manager;