Add support for GTest based unit tests. (#485)

* Add support for GTest based unit tests.

As Dominic and I have previously discussed, there is some
need/desire to improve the testing situation in Google Benchmark.

One step to fixing this problem is to make it easier to write
unit tests by adding support for GTest, which is what this patch does.

By default it looks for an installed version of GTest. However the
user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead
download, build, and use copy of gtest from source. This is
quite useful when Benchmark is being built in non-standard configurations,
such as against libc++ or in 32 bit mode.

.gitignore

@@ -44,3 +44,7 @@ rules.ninja
 # out-of-source build top-level folders.
 build/
 _build/
+
+# in-source dependancies
+/googletest/
+

.travis.yml

@@ -157,7 +157,7 @@ install:
     fi
 
 script:
-  - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ..
+  - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ..
   - make
   - ctest -C ${BUILD_TYPE} --output-on-failure
 

CMakeLists.txt

@@ -5,6 +5,7 @@ project (benchmark)
 foreach(p
     CMP0054 # CMake 3.1
     CMP0056 # export EXE_LINKER_FLAGS to try_run
+    CMP0057 # Support no if() IN_LIST operator
     )
   if(POLICY ${p})
     cmake_policy(SET ${p} NEW)
@@ -18,6 +19,14 @@ option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library
 option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF)
 option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON)
 
+# Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which
+# may require downloading the source code.
+option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree building of unmet dependencies" OFF)
+
+# This option can be used to disable building and running unit tests which depend on gtest
+# in cases where it is not possible to build or find a valid version of gtest.
+option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON)
+
 # Make sure we can import out CMake functions
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@@ -97,9 +106,7 @@ else()
   if (NOT BENCHMARK_ENABLE_EXCEPTIONS)
     add_cxx_compiler_flag(-fno-exceptions)
   endif()
-  if (NOT BENCHMARK_USE_LIBCXX)
-    add_cxx_compiler_flag(-Wzero-as-null-pointer-constant)
-  endif()
+
   if (HAVE_CXX_FLAG_FSTRICT_ALIASING)
     if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing
       add_cxx_compiler_flag(-Wstrict-aliasing)
@@ -199,5 +206,8 @@ add_subdirectory(src)
 
 if (BENCHMARK_ENABLE_TESTING)
   enable_testing()
+  if (BENCHMARK_ENABLE_GTEST_TESTS)
+    include(HandleGTest)
+  endif()
   add_subdirectory(test)
 endif()

README.md

@@ -14,6 +14,31 @@ IRC channel: https://freenode.net #googlebenchmark
 [Additional Tooling Documentation](docs/tools.md)
 
 
+## Building
+
+The basic steps for configuring and building the library look like this:
+
+```bash
+$ git clone https://github.com/google/benchmark.git
+# Benchmark requires GTest as a dependency. Add the source tree as a subdirectory.
+$ git clone https://github.com/google/googletest.git benchmark/googletest
+$ mkdir build && cd build
+$ cmake -G <generator> [options] ../benchmark
+# Assuming a makefile generator was used
+$ make
+```
+
+Note that Google Benchmark requires GTest to build and run the tests. This
+dependency can be provided three ways:
+
+* Checkout the GTest sources into `benchmark/googletest`.
+* Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during
+  configuration, the library will automatically download and build any required
+  dependencies.
+* Otherwise, if nothing is done, CMake will use `find_package(GTest REQUIRED)`
+  to resolve the required GTest dependency.
+
+
 ## Installation Guide
 
 For Ubuntu and Debian Based System

appveyor.yml

@@ -43,7 +43,7 @@ build_script:
   - md _build -Force
   - cd _build
   - echo %configuration%
-  - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" ..
+  - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON ..
   - cmake --build . --config %configuration%
 
 test_script:

cmake/HandleGTest.cmake

@@ -0,0 +1,77 @@
+
+macro(split_list listname)
+  string(REPLACE ";" " " ${listname} "${${listname}}")
+endmacro()
+
+macro(build_external_gtest)
+  include(ExternalProject)
+  set(GTEST_FLAGS "")
+  if (BENCHMARK_USE_LIBCXX)
+    if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+      list(APPEND GTEST_FLAGS -stdlib=libc++)
+    else()
+      message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++")
+    endif()
+  endif()
+  if (BENCHMARK_BUILD_32_BITS)
+    list(APPEND GTEST_FLAGS -m32)
+  endif()
+  if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "")
+    list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS})
+  endif()
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE)
+  if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE")
+    set(GTEST_BUILD_TYPE "DEBUG")
+  endif()
+  split_list(GTEST_FLAGS)
+  ExternalProject_Add(googletest
+      EXCLUDE_FROM_ALL ON
+      GIT_REPOSITORY https://github.com/google/googletest.git
+      GIT_TAG master
+      PREFIX "${CMAKE_BINARY_DIR}/googletest"
+      INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest"
+      CMAKE_CACHE_ARGS
+        -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE}
+        -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
+        -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+        -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS}
+        -Dgtest_force_shared_crt:BOOL=ON
+      )
+
+  ExternalProject_Get_Property(googletest install_dir)
+
+  add_library(gtest UNKNOWN IMPORTED)
+  add_library(gtest_main UNKNOWN IMPORTED)
+
+  set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}")
+
+  if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG")
+    set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  endif()
+  file(MAKE_DIRECTORY ${install_dir}/include)
+  set_target_properties(gtest PROPERTIES
+    IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest${LIB_SUFFIX}
+    INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include
+  )
+  set_target_properties(gtest_main PROPERTIES
+    IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest_main${LIB_SUFFIX}
+    INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include
+  )
+  add_dependencies(gtest googletest)
+  add_dependencies(gtest_main googletest)
+  set(GTEST_BOTH_LIBRARIES gtest gtest_main)
+  #set(GTEST_INCLUDE_DIRS ${install_dir}/include)
+endmacro(build_external_gtest)
+
+if (BENCHMARK_ENABLE_GTEST_TESTS)
+  if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest)
+    add_subdirectory(${CMAKE_SOURCE_DIR}/googletest)
+    set(GTEST_BOTH_LIBRARIES gtest gtest_main)
+  elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES)
+    build_external_gtest()
+  else()
+    find_package(GTest REQUIRED)
+  endif()
+endif()

test/CMakeLists.txt

@@ -42,7 +42,6 @@ macro(compile_output_test name)
           ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 endmacro(compile_output_test)
 
-
 # Demonstration executable
 compile_benchmark_test(benchmark_test)
 add_test(benchmark benchmark_test --benchmark_min_time=0.01)
@@ -135,6 +134,29 @@ endif()
 compile_output_test(complexity_test)
 add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME})
 
+###############################################################################
+# GoogleTest Unit Tests
+###############################################################################
+
+if (BENCHMARK_ENABLE_GTEST_TESTS)
+  macro(compile_gtest name)
+    add_executable(${name} "${name}.cc")
+    if (TARGET googletest)
+      add_dependencies(${name} googletest)
+    endif()
+    target_link_libraries(${name} benchmark
+        "${GTEST_BOTH_LIBRARIES}" ${CMAKE_THREAD_LIBS_INIT})
+  endmacro(compile_gtest)
+
+  macro(add_gtest name)
+    compile_gtest(${name})
+    add_test(${name} ${name})
+  endmacro()
+
+  add_gtest(statistics_test)
+endif(BENCHMARK_ENABLE_GTEST_TESTS)
+
+
 # Add the coverage command(s)
 if(CMAKE_BUILD_TYPE)
   string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)

test/statistics_test.cc

@@ -0,0 +1,61 @@
+//===---------------------------------------------------------------------===//
+// statistics_test - Unit tests for src/statistics.cc
+//===---------------------------------------------------------------------===//
+
+#include "../src/statistics.h"
+#include "gtest/gtest.h"
+
+namespace {
+TEST(StatisticsTest, Mean) {
+  std::vector<double> Inputs;
+  {
+    Inputs = {42, 42, 42, 42};
+    double Res = benchmark::StatisticsMean(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 42.0);
+  }
+  {
+    Inputs = {1, 2, 3, 4};
+    double Res = benchmark::StatisticsMean(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 2.5);
+  }
+  {
+    Inputs = {1, 2, 5, 10, 10, 14};
+    double Res = benchmark::StatisticsMean(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 7.0);
+  }
+}
+
+TEST(StatisticsTest, Median) {
+  std::vector<double> Inputs;
+  {
+    Inputs = {42, 42, 42, 42};
+    double Res = benchmark::StatisticsMedian(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 42.0);
+  }
+  {
+    Inputs = {1, 2, 3, 4};
+    double Res = benchmark::StatisticsMedian(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 2.5);
+  }
+  {
+    Inputs = {1, 2, 5, 10, 10};
+    double Res = benchmark::StatisticsMedian(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 5.0);
+  }
+}
+
+TEST(StatisticsTest, StdDev) {
+  std::vector<double> Inputs;
+  {
+    Inputs = {101, 101, 101, 101};
+    double Res = benchmark::StatisticsStdDev(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 0.0);
+  }
+  {
+    Inputs = {1, 2, 3};
+    double Res = benchmark::StatisticsStdDev(Inputs);
+    EXPECT_DOUBLE_EQ(Res, 1.0);
+  }
+}
+
+}  // end namespace