From de725e5a7c788311e802f5ed0c3763331a9db60b Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 13 Dec 2017 14:51:56 -0700 Subject: [PATCH 1/2] Document new 'v2' branch meant for unstable development. This patch documents the newly added v2 branch, which will be used to stage, test, and receive feedback on upcoming features, most of which will be breaking changes which can't be directly applied to master. --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 68796305..baa99326 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,18 @@ sudo make install Now you have google/benchmark installed in your machine Note: Don't forget to link to pthread library while building +## Stable and Experimental Library Versions + +The main branch contains the latest stable version of the benchmarking library; +the API of which can be considered largely stable, with source breaking changes +being made only upon the release of a new major version. + +Newer, experimental, features are implemented and tested on the +[`v2` branch](https://github.com/google/benchmark/tree/v2). Users who wish +to use, test, and provide feedback on the new features are encouraged to try +this branch. However, this branch provides no stability guarantees and reserves +the right to change and break the API at any time. + ## Example usage ### Basic usage From 7db02be244cb2a960c3da48cbdb5018c7a72b5d7 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 13 Dec 2017 16:26:47 -0700 Subject: [PATCH 2/2] Add support for GTest based unit tests. (#485) * Add support for GTest based unit tests. As Dominic and I have previously discussed, there is some need/desire to improve the testing situation in Google Benchmark. One step to fixing this problem is to make it easier to write unit tests by adding support for GTest, which is what this patch does. By default it looks for an installed version of GTest. However the user can specify -DBENCHMARK_BUILD_EXTERNAL_GTEST=ON to instead download, build, and use copy of gtest from source. This is quite useful when Benchmark is being built in non-standard configurations, such as against libc++ or in 32 bit mode. --- .gitignore | 4 +++ .travis.yml | 2 +- CMakeLists.txt | 16 +++++++-- README.md | 25 +++++++++++++ appveyor.yml | 2 +- cmake/HandleGTest.cmake | 77 +++++++++++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 24 ++++++++++++- test/statistics_test.cc | 61 ++++++++++++++++++++++++++++++++ 8 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 cmake/HandleGTest.cmake create mode 100644 test/statistics_test.cc diff --git a/.gitignore b/.gitignore index 3c1b4f21..bfc6b57b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ rules.ninja # out-of-source build top-level folders. build/ _build/ + +# in-source dependancies +/googletest/ + diff --git a/.travis.yml b/.travis.yml index 1e439558..58be73f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -157,7 +157,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 44acc288..aa082676 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,7 @@ project (benchmark) foreach(p CMP0054 # CMake 3.1 CMP0056 # export EXE_LINKER_FLAGS to try_run + CMP0057 # Support no if() IN_LIST operator ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) @@ -18,6 +19,14 @@ option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON) +# Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which +# may require downloading the source code. +option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree building of unmet dependencies" OFF) + +# This option can be used to disable building and running unit tests which depend on gtest +# in cases where it is not possible to build or find a valid version of gtest. +option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) + # Make sure we can import out CMake functions list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -97,9 +106,7 @@ else() if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() - if (NOT BENCHMARK_USE_LIBCXX) - add_cxx_compiler_flag(-Wzero-as-null-pointer-constant) - endif() + if (HAVE_CXX_FLAG_FSTRICT_ALIASING) if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing add_cxx_compiler_flag(-Wstrict-aliasing) @@ -199,5 +206,8 @@ add_subdirectory(src) if (BENCHMARK_ENABLE_TESTING) enable_testing() + if (BENCHMARK_ENABLE_GTEST_TESTS) + include(HandleGTest) + endif() add_subdirectory(test) endif() diff --git a/README.md b/README.md index baa99326..45e27ed5 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,31 @@ IRC channel: https://freenode.net #googlebenchmark [Additional Tooling Documentation](docs/tools.md) +## Building + +The basic steps for configuring and building the library look like this: + +```bash +$ git clone https://github.com/google/benchmark.git +# Benchmark requires GTest as a dependency. Add the source tree as a subdirectory. +$ git clone https://github.com/google/googletest.git benchmark/googletest +$ mkdir build && cd build +$ cmake -G [options] ../benchmark +# Assuming a makefile generator was used +$ make +``` + +Note that Google Benchmark requires GTest to build and run the tests. This +dependency can be provided three ways: + +* Checkout the GTest sources into `benchmark/googletest`. +* Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during + configuration, the library will automatically download and build any required + dependencies. +* Otherwise, if nothing is done, CMake will use `find_package(GTest REQUIRED)` + to resolve the required GTest dependency. + + ## Installation Guide For Ubuntu and Debian Based System diff --git a/appveyor.yml b/appveyor.yml index e084f386..e99c6e77 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,7 +43,7 @@ build_script: - md _build -Force - cd _build - echo %configuration% - - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" .. + - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON .. - cmake --build . --config %configuration% test_script: diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake new file mode 100644 index 00000000..c84f6539 --- /dev/null +++ b/cmake/HandleGTest.cmake @@ -0,0 +1,77 @@ + +macro(split_list listname) + string(REPLACE ";" " " ${listname} "${${listname}}") +endmacro() + +macro(build_external_gtest) + include(ExternalProject) + set(GTEST_FLAGS "") + if (BENCHMARK_USE_LIBCXX) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + list(APPEND GTEST_FLAGS -stdlib=libc++) + else() + message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++") + endif() + endif() + if (BENCHMARK_BUILD_32_BITS) + list(APPEND GTEST_FLAGS -m32) + endif() + if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "") + list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS}) + endif() + string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE) + if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE") + set(GTEST_BUILD_TYPE "DEBUG") + endif() + split_list(GTEST_FLAGS) + ExternalProject_Add(googletest + EXCLUDE_FROM_ALL ON + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + PREFIX "${CMAKE_BINARY_DIR}/googletest" + INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest" + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE} + -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_PREFIX:PATH= + -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS} + -Dgtest_force_shared_crt:BOOL=ON + ) + + ExternalProject_Get_Property(googletest install_dir) + + add_library(gtest UNKNOWN IMPORTED) + add_library(gtest_main UNKNOWN IMPORTED) + + set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}") + + if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG") + set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + file(MAKE_DIRECTORY ${install_dir}/include) + set_target_properties(gtest PROPERTIES + IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest${LIB_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include + ) + set_target_properties(gtest_main PROPERTIES + IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest_main${LIB_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include + ) + add_dependencies(gtest googletest) + add_dependencies(gtest_main googletest) + set(GTEST_BOTH_LIBRARIES gtest gtest_main) + #set(GTEST_INCLUDE_DIRS ${install_dir}/include) +endmacro(build_external_gtest) + +if (BENCHMARK_ENABLE_GTEST_TESTS) + if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest) + add_subdirectory(${CMAKE_SOURCE_DIR}/googletest) + set(GTEST_BOTH_LIBRARIES gtest gtest_main) + elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) + build_external_gtest() + else() + find_package(GTest REQUIRED) + endif() +endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index baf21cbc..efce3ba5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -42,7 +42,6 @@ macro(compile_output_test name) ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) - # Demonstration executable compile_benchmark_test(benchmark_test) add_test(benchmark benchmark_test --benchmark_min_time=0.01) @@ -135,6 +134,29 @@ endif() compile_output_test(complexity_test) add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +############################################################################### +# GoogleTest Unit Tests +############################################################################### + +if (BENCHMARK_ENABLE_GTEST_TESTS) + macro(compile_gtest name) + add_executable(${name} "${name}.cc") + if (TARGET googletest) + add_dependencies(${name} googletest) + endif() + target_link_libraries(${name} benchmark + "${GTEST_BOTH_LIBRARIES}" ${CMAKE_THREAD_LIBS_INIT}) + endmacro(compile_gtest) + + macro(add_gtest name) + compile_gtest(${name}) + add_test(${name} ${name}) + endmacro() + + add_gtest(statistics_test) +endif(BENCHMARK_ENABLE_GTEST_TESTS) + + # Add the coverage command(s) if(CMAKE_BUILD_TYPE) string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) diff --git a/test/statistics_test.cc b/test/statistics_test.cc new file mode 100644 index 00000000..b4d6abbb --- /dev/null +++ b/test/statistics_test.cc @@ -0,0 +1,61 @@ +//===---------------------------------------------------------------------===// +// statistics_test - Unit tests for src/statistics.cc +//===---------------------------------------------------------------------===// + +#include "../src/statistics.h" +#include "gtest/gtest.h" + +namespace { +TEST(StatisticsTest, Mean) { + std::vector Inputs; + { + Inputs = {42, 42, 42, 42}; + double Res = benchmark::StatisticsMean(Inputs); + EXPECT_DOUBLE_EQ(Res, 42.0); + } + { + Inputs = {1, 2, 3, 4}; + double Res = benchmark::StatisticsMean(Inputs); + EXPECT_DOUBLE_EQ(Res, 2.5); + } + { + Inputs = {1, 2, 5, 10, 10, 14}; + double Res = benchmark::StatisticsMean(Inputs); + EXPECT_DOUBLE_EQ(Res, 7.0); + } +} + +TEST(StatisticsTest, Median) { + std::vector Inputs; + { + Inputs = {42, 42, 42, 42}; + double Res = benchmark::StatisticsMedian(Inputs); + EXPECT_DOUBLE_EQ(Res, 42.0); + } + { + Inputs = {1, 2, 3, 4}; + double Res = benchmark::StatisticsMedian(Inputs); + EXPECT_DOUBLE_EQ(Res, 2.5); + } + { + Inputs = {1, 2, 5, 10, 10}; + double Res = benchmark::StatisticsMedian(Inputs); + EXPECT_DOUBLE_EQ(Res, 5.0); + } +} + +TEST(StatisticsTest, StdDev) { + std::vector Inputs; + { + Inputs = {101, 101, 101, 101}; + double Res = benchmark::StatisticsStdDev(Inputs); + EXPECT_DOUBLE_EQ(Res, 0.0); + } + { + Inputs = {1, 2, 3}; + double Res = benchmark::StatisticsStdDev(Inputs); + EXPECT_DOUBLE_EQ(Res, 1.0); + } +} + +} // end namespace