Merge branch 'main' into benchmark_list_tests-work-with-benchmark-format=json-#1642

AUTHORS

@@ -31,6 +31,7 @@ Evgeny Safronov <division494@gmail.com>
 Fabien Pichot <pichot.fabien@gmail.com>
 Federico Ficarelli <federico.ficarelli@gmail.com>
 Felix Homann <linuxaudio@showlabor.de>
+Gergely Meszaros <maetveis@gmail.com>
 Gergő Szitár <szitar.gergo@gmail.com>
 Google Inc.
 Henrique Bucher <hbucher@gmail.com>

CMakeLists.txt

@@ -1,7 +1,7 @@
 # Require CMake 3.10. If available, use the policies up to CMake 3.22.
 cmake_minimum_required (VERSION 3.10...3.22)
 
-project (benchmark VERSION 1.8.2 LANGUAGES CXX)
+project (benchmark VERSION 1.8.3 LANGUAGES CXX)
 
 option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
 option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON)
@@ -21,7 +21,7 @@ if(BENCHMARK_FORCE_WERROR)
   set(BENCHMARK_ENABLE_WERROR ON)
 endif(BENCHMARK_FORCE_WERROR)
 
-if(NOT MSVC)
+if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC"))
   option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF)
 else()
   set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE)
@@ -45,7 +45,7 @@ option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm"
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
 
-if(MSVC)
+if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
     # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and
     # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the
     # undocumented, but working variable.
@@ -66,7 +66,7 @@ function(should_enable_assembly_tests)
       return()
     endif()
   endif()
-  if (MSVC)
+  if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
     return()
   elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
     return()
@@ -128,7 +128,7 @@ if (BENCHMARK_BUILD_32_BITS)
   add_required_cxx_compiler_flag(-m32)
 endif()
 
-if (MSVC)
+if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
   set(BENCHMARK_CXX_STANDARD 14)
 else()
   set(BENCHMARK_CXX_STANDARD 11)

CONTRIBUTORS

@@ -51,6 +51,7 @@ Fanbo Meng <fanbo.meng@ibm.com>
 Federico Ficarelli <federico.ficarelli@gmail.com>
 Felix Homann <linuxaudio@showlabor.de>
 Geoffrey Martin-Noble <gcmn@google.com> <gmngeoffrey@gmail.com>
+Gergely Meszaros <maetveis@gmail.com>
 Gergő Szitár <szitar.gergo@gmail.com>
 Hannes Hauswedell <h2@fsfe.org>
 Henrique Bucher <hbucher@gmail.com>

MODULE.bazel

@@ -1,4 +1,4 @@
-module(name = "google_benchmark", version="1.8.2")
+module(name = "google_benchmark", version="1.8.3")
 
 bazel_dep(name = "bazel_skylib", version = "1.4.1")
 bazel_dep(name = "platforms", version = "0.0.6")

bindings/python/google_benchmark/__init__.py

@@ -69,7 +69,7 @@ __all__ = [
     "State",
 ]
 
-__version__ = "1.8.2"
+__version__ = "1.8.3"
 
 
 class __OptionMaker:

docs/_config.yml

@@ -1 +1,3 @@
 theme: jekyll-theme-minimal
+logo: /assets/images/icon_black.png
+show_downloads: true

src/benchmark.cc

@@ -179,6 +179,17 @@ State::State(std::string name, IterationCount max_iters,
   BM_CHECK_LT(thread_index_, threads_)
       << "thread_index must be less than threads";
 
+  // Add counters with correct flag now.  If added with `counters[name]` in
+  // `PauseTiming`, a new `Counter` will be inserted the first time, which
+  // won't have the flag.  Inserting them now also reduces the allocations
+  // during the benchmark.
+  if (perf_counters_measurement_) {
+    for (const std::string& counter_name :
+         perf_counters_measurement_->names()) {
+      counters[counter_name] = Counter(0.0, Counter::kAvgIterations);
+    }
+  }
+
   // Note: The use of offsetof below is technically undefined until C++17
   // because State is not a standard layout type. However, all compilers
   // currently provide well-defined behavior as an extension (which is
@@ -227,9 +238,11 @@ void State::PauseTiming() {
       BM_CHECK(false) << "Perf counters read the value failed.";
     }
     for (const auto& name_and_measurement : measurements) {
-      auto name = name_and_measurement.first;
-      auto measurement = name_and_measurement.second;
-      counters[name] += Counter(measurement, Counter::kAvgIterations);
+      const std::string& name = name_and_measurement.first;
+      const double measurement = name_and_measurement.second;
+      // Counter was inserted with `kAvgIterations` flag by the constructor.
+      assert(counters.find(name) != counters.end());
+      counters[name].value += measurement;
     }
   }
 }
@@ -383,7 +396,7 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
       BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr;
       if (benchmark.complexity() != oNone)
         reports_for_family = &per_family_reports[benchmark.family_index()];
-      benchmarks_with_threads += (benchmark.threads() > 0);
+      benchmarks_with_threads += (benchmark.threads() > 1);
       runners.emplace_back(benchmark, &perfcounters, reports_for_family);
       int num_repeats_of_this_instance = runners.back().GetNumRepeats();
       num_repetitions_total += num_repeats_of_this_instance;

src/perf_counters.cc

@@ -57,9 +57,18 @@ size_t PerfCounterValues::Read(const std::vector<int>& leaders) {
 
 const bool PerfCounters::kSupported = true;
 
-bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; }
+// Initializes libpfm only on the first call.  Returns whether that single
+// initialization was successful.
+bool PerfCounters::Initialize() {
+  // Function-scope static gets initialized only once on first call.
+  static const bool success = []() {
+    return pfm_initialize() == PFM_SUCCESS;
+  }();
+  return success;
+}
 
 bool PerfCounters::IsCounterSupported(const std::string& name) {
+  Initialize();
   perf_event_attr_t attr;
   std::memset(&attr, 0, sizeof(attr));
   pfm_perf_encode_arg_t arg;
@@ -73,6 +82,10 @@ bool PerfCounters::IsCounterSupported(const std::string& name) {
 
 PerfCounters PerfCounters::Create(
     const std::vector<std::string>& counter_names) {
+  if (!counter_names.empty()) {
+    Initialize();
+  }
+
   // Valid counters will populate these arrays but we start empty
   std::vector<std::string> valid_names;
   std::vector<int> counter_ids;

src/perf_counters.h

@@ -190,8 +190,6 @@ class BENCHMARK_EXPORT PerfCountersMeasurement final {
   PerfCounterValues end_values_;
 };
 
-BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize();
-
 }  // namespace internal
 }  // namespace benchmark
 

src/statistics.cc

@@ -42,13 +42,13 @@ double StatisticsMedian(const std::vector<double>& v) {
   auto center = copy.begin() + v.size() / 2;
   std::nth_element(copy.begin(), center, copy.end());
 
-  // did we have an odd number of samples?
-  // if yes, then center is the median
-  // it no, then we are looking for the average between center and the value
-  // before
+  // Did we have an odd number of samples?  If yes, then center is the median.
+  // If not, then we are looking for the average between center and the value
+  // before.  Instead of resorting, we just look for the max value before it,
+  // which is not necessarily the element immediately preceding `center` Since
+  // `copy` is only partially sorted by `nth_element`.
   if (v.size() % 2 == 1) return *center;
-  auto center2 = copy.begin() + v.size() / 2 - 1;
-  std::nth_element(copy.begin(), center2, copy.end());
+  auto center2 = std::max_element(copy.begin(), center);
   return (*center + *center2) / 2.0;
 }
 

src/sysinfo.cc

@@ -817,7 +817,7 @@ std::vector<double> GetLoadAvg() {
 #if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) ||     \
      defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD ||      \
      defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \
-    !defined(__ANDROID__)
+    !(defined(__ANDROID__) && __ANDROID_API__ < 29)
   static constexpr int kMaxSamples = 3;
   std::vector<double> res(kMaxSamples, 0.0);
   const int nelem = getloadavg(res.data(), kMaxSamples);

test/BUILD

@@ -49,6 +49,27 @@ cc_library(
     ],
 )
 
+# Tests that use gtest.  These rely on `gtest_main`.
+[
+    cc_test(
+        name = test_src[:-len(".cc")],
+        size = "small",
+        srcs = [test_src],
+        copts = select({
+            "//:windows": [],
+            "//conditions:default": TEST_COPTS,
+        }) + PER_SRC_COPTS.get(test_src, []),
+        deps = [
+            "//:benchmark",
+            "//:benchmark_internal_headers",
+            "@com_google_googletest//:gtest",
+            "@com_google_googletest//:gtest_main",
+        ],
+    )
+    for test_src in glob(["*_gtest.cc"])
+]
+
+# Tests that do not use gtest.  These have their own `main` defined.
 [
     cc_test(
         name = test_src[:-len(".cc")],
@@ -63,15 +84,13 @@ cc_library(
             ":output_test_helper",
             "//:benchmark",
             "//:benchmark_internal_headers",
-            "@com_google_googletest//:gtest",
-            "@com_google_googletest//:gtest_main",
         ],
         # FIXME: Add support for assembly tests to bazel.
         # See Issue #556
         # https://github.com/google/benchmark/issues/556
     )
     for test_src in glob(
-        ["*test.cc"],
+        ["*_test.cc"],
         exclude = [
             "*_assembly_test.cc",
             "cxx03_test.cc",
@@ -93,8 +112,6 @@ cc_test(
         ":output_test_helper",
         "//:benchmark",
         "//:benchmark_internal_headers",
-        "@com_google_googletest//:gtest",
-        "@com_google_googletest//:gtest_main",
     ],
 )
 

test/CMakeLists.txt

@@ -183,7 +183,7 @@ compile_output_test(memory_manager_test)
 add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s)
 
 # MSVC does not allow to set the language standard to C++98/03.
-if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC"))
   compile_benchmark_test(cxx03_test)
   set_target_properties(cxx03_test
       PROPERTIES

test/perf_counters_gtest.cc

@@ -21,8 +21,7 @@ using ::testing::Lt;
 
 namespace {
 const char kGenericPerfEvent1[] = "CYCLES";
-const char kGenericPerfEvent2[] = "BRANCHES";
-const char kGenericPerfEvent3[] = "INSTRUCTIONS";
+const char kGenericPerfEvent2[] = "INSTRUCTIONS";
 
 TEST(PerfCountersTest, Init) {
   EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported);
@@ -61,26 +60,24 @@ TEST(PerfCountersTest, NegativeTest) {
   {
     // Try sneaking in an outrageous counter, like a fat finger mistake
     auto counter = PerfCounters::Create(
-        {kGenericPerfEvent3, "not a counter name", kGenericPerfEvent1});
+        {kGenericPerfEvent2, "not a counter name", kGenericPerfEvent1});
     EXPECT_EQ(counter.num_counters(), 2);
     EXPECT_EQ(counter.names(), std::vector<std::string>(
-                                   {kGenericPerfEvent3, kGenericPerfEvent1}));
+                                   {kGenericPerfEvent2, kGenericPerfEvent1}));
   }
   {
-    // Finally try a golden input - it should like all them
-    EXPECT_EQ(PerfCounters::Create(
-                  {kGenericPerfEvent1, kGenericPerfEvent2, kGenericPerfEvent3})
+    // Finally try a golden input - it should like both of them
+    EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2})
                   .num_counters(),
-              3);
+              2);
   }
   {
     // Add a bad apple in the end of the chain to check the edges
-    auto counter = PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2,
-                                         kGenericPerfEvent3, "bad event name"});
-    EXPECT_EQ(counter.num_counters(), 3);
-    EXPECT_EQ(counter.names(),
-              std::vector<std::string>({kGenericPerfEvent1, kGenericPerfEvent2,
-                                        kGenericPerfEvent3}));
+    auto counter = PerfCounters::Create(
+        {kGenericPerfEvent1, kGenericPerfEvent2, "bad event name"});
+    EXPECT_EQ(counter.num_counters(), 2);
+    EXPECT_EQ(counter.names(), std::vector<std::string>(
+                                   {kGenericPerfEvent1, kGenericPerfEvent2}));
   }
 }
 
@@ -119,26 +116,25 @@ TEST(PerfCountersTest, Read2Counters) {
 }
 
 TEST(PerfCountersTest, ReopenExistingCounters) {
-  // This test works in recent and old Intel hardware
-  // However we cannot make assumptions beyond 3 HW counters
+  // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6.
+  // However we cannot make assumptions beyond 2 HW counters due to Pixel 6.
   if (!PerfCounters::kSupported) {
     GTEST_SKIP() << "Test skipped because libpfm is not supported.\n";
   }
   EXPECT_TRUE(PerfCounters::Initialize());
   std::vector<std::string> kMetrics({kGenericPerfEvent1});
-  std::vector<PerfCounters> counters(3);
+  std::vector<PerfCounters> counters(2);
   for (auto& counter : counters) {
     counter = PerfCounters::Create(kMetrics);
   }
   PerfCounterValues values(1);
   EXPECT_TRUE(counters[0].Snapshot(&values));
   EXPECT_TRUE(counters[1].Snapshot(&values));
-  EXPECT_TRUE(counters[2].Snapshot(&values));
 }
 
 TEST(PerfCountersTest, CreateExistingMeasurements) {
   // The test works (i.e. causes read to fail) for the assumptions
-  // about hardware capabilities (i.e. small number (3) hardware
+  // about hardware capabilities (i.e. small number (2) hardware
   // counters) at this date,
   // the same as previous test ReopenExistingCounters.
   if (!PerfCounters::kSupported) {
@@ -151,7 +147,7 @@ TEST(PerfCountersTest, CreateExistingMeasurements) {
   // we could use libpfm to query for the hardware limits on this
   // particular platform.
   const int kMaxCounters = 10;
-  const int kMinValidCounters = 3;
+  const int kMinValidCounters = 2;
 
   // Let's use a ubiquitous counter that is guaranteed to work
   // on all platforms
@@ -229,7 +225,7 @@ void measure(size_t threadcount, PerfCounterValues* before,
   // the scopes overlap, and we need to explicitly control the scope of the
   // threadpool.
   auto counters =
-      PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3});
+      PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2});
   for (auto& t : threads) t = std::thread(work);
   counters.Snapshot(before);
   for (auto& t : threads) t.join();
@@ -281,16 +277,14 @@ TEST(PerfCountersTest, HardwareLimits) {
   EXPECT_TRUE(PerfCounters::Initialize());
 
   // Taken from `perf list`, but focusses only on those HW events that actually
-  // were reported when running `sudo perf stat -a sleep 10`. All HW events
-  // listed in the first command not reported in the second seem to not work.
-  // This is sad as we don't really get to test the grouping here (groups can
-  // contain up to 6 members)...
+  // were reported when running `sudo perf stat -a sleep 10`, intersected over
+  // several platforms. All HW events listed in the first command not reported
+  // in the second seem to not work. This is sad as we don't really get to test
+  // the grouping here (groups can contain up to 6 members)...
   std::vector<std::string> counter_names{
       "cycles",         // leader
       "instructions",   //
-      "branches",       //
       "branch-misses",  //
-      "cache-misses",   //
   };
 
   // In the off-chance that some of these values are not supported,

test/perf_counters_test.cc

@@ -61,34 +61,28 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithPauseResume\",$"}});
 
 static void CheckSimple(Results const& e) {
   CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0);
-  CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0);
 }
 
 double withoutPauseResumeInstrCount = 0.0;
 double withPauseResumeInstrCount = 0.0;
 
-static void CheckInstrCount(double* counter, Results const& e) {
-  BM_CHECK_GT(e.NumIterations(), 0);
-  *counter = e.GetAs<double>("INSTRUCTIONS") / e.NumIterations();
+static void SaveInstrCountWithoutResume(Results const& e) {
+  withoutPauseResumeInstrCount = e.GetAs<double>("INSTRUCTIONS");
 }
 
-static void CheckInstrCountWithoutResume(Results const& e) {
-  CheckInstrCount(&withoutPauseResumeInstrCount, e);
-}
-
-static void CheckInstrCountWithResume(Results const& e) {
-  CheckInstrCount(&withPauseResumeInstrCount, e);
+static void SaveInstrCountWithResume(Results const& e) {
+  withPauseResumeInstrCount = e.GetAs<double>("INSTRUCTIONS");
 }
 
 CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple);
-CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &CheckInstrCountWithoutResume);
-CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &CheckInstrCountWithResume);
+CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &SaveInstrCountWithoutResume);
+CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &SaveInstrCountWithResume);
 
 int main(int argc, char* argv[]) {
   if (!benchmark::internal::PerfCounters::kSupported) {
     return 0;
   }
-  benchmark::FLAGS_benchmark_perf_counters = "CYCLES,BRANCHES,INSTRUCTIONS";
+  benchmark::FLAGS_benchmark_perf_counters = "CYCLES,INSTRUCTIONS";
   benchmark::internal::PerfCounters::Initialize();
   RunOutputTests(argc, argv);