Integrate code coverage with Apollo

Reviewers: teon.banek

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1050

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.1)
 # to download dependencies
 
 if(NOT UNIX)
-    message(FATAL "Unsupported operating system.")
+  message(FATAL_ERROR "Unsupported operating system.")
 endif()
 
 # Set `make clean` to ignore outputs of add_custom_command. If generated files
@@ -171,17 +171,26 @@ add_library(antlr_opencypher_parser_lib STATIC ${antlr_opencypher_generated_src}
 target_link_libraries(antlr_opencypher_parser_lib antlr4)
 # -----------------------------------------------------------------------------
 
-include_directories(src)
-add_subdirectory(src)
-# -----------------------------------------------------------------------------
-
 # Optional subproject configuration -------------------------------------------
 option(POC "Build proof of concept binaries" ON)
 option(EXPERIMENTAL "Build experimental binaries" OFF)
 option(CUSTOMERS "Build customer binaries" ON)
-option(TEST_COVERAGE "Generate coverage reports from unit tests" OFF)
+option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
 option(TOOLS "Build tools binaries" OFF)
 
+if (TEST_COVERAGE)
+  string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
+  if (NOT lower_build_type STREQUAL "debug")
+    message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+endif()
+
+# Add subprojects
+include_directories(src)
+add_subdirectory(src)
+
 if(POC)
   add_subdirectory(poc)
 endif()

apollo_build.yaml

@@ -9,9 +9,16 @@
     doxygen Doxyfile
 
     cd build
-    cmake -DTOOLS=ON -DTEST_COVERAGE=ON ..
+    cmake -DTOOLS=ON ..
     TIMEOUT=1000 make -j$THREADS
 
+    cd ..
+    mkdir build_coverage
+
+    cd build_coverage
+    cmake -DTEST_COVERAGE=ON ..
+    TIMEOUT=1000 make -j$THREADS memgraph__unit
+
     cd ..
     mkdir build_release
 

coverage

@@ -1,59 +0,0 @@
-#!/bin/bash
-
-working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# coverage applies only on unit tests
-unit_test_dir='build/tests/unit'
-coverage_dir='build/coverage'
-coverage_file='coverage.info'
-
-# execute unit tests to generate coverage files
-pushd ${unit_test_dir}
-ctest -R unit --quiet
-popd
-
-# generate coverage info files for each unit test binary
-#   + exclude stl, tests and libs
-cd ${working_dir}
-binary_path="${unit_test_dir}/CMakeFiles"
-all_coverage_info=""
-for dir in $binary_path/*.dir; do
-    pushd ${dir}
-    lcov --gcov-tool ${working_dir}/llvm-gcov -c -d . -o ${coverage_file}
-    lcov -r ${coverage_file} '/usr/*' '*/libs/*' '*/tests/*' -o ${coverage_file}
-    test_coverage_file=${working_dir}/${dir}/${coverage_file}
-    if [ -f ${coverage_file} ]; then
-        if [ -s ${coverage_file} ]; then
-            char_no=`wc -c ${coverage_file} | awk '{print $1}'`
-            if [ "${char_no}" -eq "0" ]; then
-                echo "PASS: ${test_coverage_file} contains 0 chars."
-                popd
-                continue
-            fi
-        else
-            echo "PASS: ${test_coverage_file} is empty."
-            popd
-            continue
-        fi
-    else
-        echo "PASS: ${test_coverage_file} doesn't exist."
-        popd
-        continue
-    fi
-    all_coverage_info+=" -a ${working_dir}/${dir}/${coverage_file}"
-    popd
-done
-
-# merge all info files into total coverage info and generate html
-echo ${all_coverage_info}
-cd ${working_dir}
-mkdir -p ${coverage_dir}
-pushd ${coverage_dir}
-lcov ${all_coverage_info} -o ${coverage_file}
-genhtml ${coverage_file}
-popd
-
-# generage coberatura xml file (compatible with Jenkins)
-cd ${working_dir}
-python libs/lcov-to-cobertura-xml/lcov_cobertura/lcov_cobertura.py \
-    ${coverage_dir}/${coverage_file} -o ${coverage_dir}/coverage.xml

experimental/distributed/tests/CMakeLists.txt

@@ -17,10 +17,6 @@ foreach(test_cpp ${test_type_cpps})
 
     # build exec file
     add_executable(${target_name} ${test_cpp})
-    if(${TEST_COVERAGE})
-        set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage")
-        set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage")
-    endif()
 
     # OUTPUT_NAME sets the real name of a target when it is built and can be
     # used to help create two targets of the same name even though CMake
@@ -31,10 +27,6 @@ foreach(test_cpp ${test_type_cpps})
     target_link_libraries(${target_name} distributed_lib memgraph_lib)
     # gtest
     target_link_libraries(${target_name} gtest gtest_main)
-    if(${TEST_COVERAGE})
-        # for code coverage
-        target_link_libraries(${target_name} gcov)
-    endif()
 
     # register test
     set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)

libs/setup.sh

@@ -86,11 +86,6 @@ clone git://deps.memgraph.io/googletest.git googletest $googletest_tag
 glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+)
 clone git://deps.memgraph.io/glog.git glog $glog_tag
 
-# lcov-to-coberatura-xml
-# git clone https://github.com/eriwen/lcov-to-cobertura-xml.git
-lcov_to_xml_tag="59584761cb5da4687693faec05bf3e2b74e9dde9" # Dec 6, 2016
-clone git://deps.memgraph.io/lcov-to-cobertura-xml.git lcov-to-cobertura-xml $lcov_to_xml_tag
-
 # google flags
 # git clone https://github.com/memgraph/gflags.git
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)

llvm-gcov

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-exec llvm-cov-3.8 gcov "$@"

src/CMakeLists.txt

@@ -51,6 +51,8 @@ set(memgraph_src_files
 )
 # -----------------------------------------------------------------------------
 
+string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
+
 # memgraph_lib depend on these libraries
 set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools
     antlr_opencypher_parser_lib dl glog gflags ${Boost_SERIALIZATION_LIBRARY_RELEASE})
@@ -90,8 +92,6 @@ add_custom_command(TARGET memgraph POST_BUILD
                    COMMENT Creating symlink to memgraph executable)
 
 # Strip the executable in release build.
-string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
-
 if (lower_build_type STREQUAL "release")
   add_custom_command(TARGET memgraph POST_BUILD
                      COMMAND strip -s $<TARGET_FILE:memgraph>

tests/apollo_runs.py

@@ -38,9 +38,7 @@ runs = []
 for test in tests:
     order, name, path = test
     dirname, basename = os.path.split(path)
-    cmakedir = os.path.join("CMakeFiles",
-            "memgraph" + CTEST_DELIMITER + name + ".dir")
-    files = [basename, cmakedir]
+    files = [basename]
 
     # extra files for specific tests
     if name == "unit__fswatcher":
@@ -56,10 +54,11 @@ for test in tests:
         prefix = "TIMEOUT=600 "
 
     outfile_paths = []
-    if name.startswith("unit"):
-        cmakedir_abs = os.path.join(TESTS_DIR, "unit", cmakedir)
-        cmakedir_rel = os.path.relpath(cmakedir_abs, WORKSPACE_DIR)
-        outfile_paths.append("\./" + cmakedir_rel.replace(".", "\\.") + ".+")
+    if name.startswith("unit") and mode == "diff":
+        dirname = dirname.replace("/build/", "/build_coverage/")
+        curdir_abs = os.path.normpath(os.path.join(SCRIPT_DIR, dirname))
+        curdir_rel = os.path.relpath(curdir_abs, WORKSPACE_DIR)
+        outfile_paths.append("\./" + curdir_rel.replace(".", "\\.") + "/.+")
 
     runs.append({
         "name": name,

tests/distributed/raft/CMakeLists.txt

@@ -15,10 +15,6 @@ foreach(test_cpp ${test_type_cpps})
 
     # build exec file
     add_executable(${target_name} ${test_cpp})
-    if(TEST_COVERAGE)
-        set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage")
-        set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage")
-    endif()
 
     # OUTPUT_NAME sets the real name of a target when it is built and can be
     # used to help create two targets of the same name even though CMake
@@ -27,10 +23,6 @@ foreach(test_cpp ${test_type_cpps})
 
     # link libraries
     target_link_libraries(${target_name} memgraph_lib)
-    if(TEST_COVERAGE)
-        # for code coverage
-        target_link_libraries(${target_name} gcov)
-    endif()
 
     set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)
 

tests/unit/CMakeLists.txt

@@ -5,6 +5,8 @@ get_filename_component(test_type ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 file(GLOB_RECURSE test_type_cpps *.cpp)
 message(STATUS "Available ${test_type} cpp files are: ${test_type_cpps}")
 
+add_custom_target(memgraph__unit)
+
 # for each cpp file build binary and register test
 foreach(test_cpp ${test_type_cpps})
 
@@ -15,10 +17,6 @@ foreach(test_cpp ${test_type_cpps})
 
     # build exec file
     add_executable(${target_name} ${test_cpp})
-    if(TEST_COVERAGE)
-        set_target_properties(${target_name} PROPERTIES COMPILE_FLAGS "-g -O0 -Wall -fprofile-arcs -ftest-coverage")
-        set_target_properties(${target_name} PROPERTIES LINK_FLAGS "--coverage -fprofile-arcs -ftest-coverage")
-    endif()
 
     # OUTPUT_NAME sets the real name of a target when it is built and can be
     # used to help create two targets of the same name even though CMake
@@ -29,13 +27,12 @@ foreach(test_cpp ${test_type_cpps})
     target_link_libraries(${target_name} memgraph_lib)
     # gtest
     target_link_libraries(${target_name} gtest gmock gtest_main)
-    if(TEST_COVERAGE)
-        # for code coverage
-        target_link_libraries(${target_name} gcov)
-    endif()
 
     # register test
     set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)
     add_test(${target_name} ${exec_name} --gtest_output=xml:${output_path})
 
+    # add to memgraph__unit target
+    add_dependencies(memgraph__unit ${target_name})
+
 endforeach()

tools/apollo/.gitignore

@@ -1 +1,2 @@
 .cppcheck_errors
+generated/*

tools/apollo/apollo_archives.yaml

@@ -0,0 +1,7 @@
+- name: Code coverage
+  cd: generated/html
+  run_type: data process
+  archive:
+    - .
+  filename: code_coverage.tar.gz
+  host: true

tools/apollo/apollo_runs.yaml

@@ -9,3 +9,16 @@
     - ../../.clang-format # clang-format config file
   outfile_paths:
     - \./memgraph/tools/apollo/\.cppcheck_errors
+
+- name: code_coverage
+  project: ^mg-master-diff$ # regex to match only 'mg-master-diff'
+  type: data process
+  commands: ./coverage_convert
+  infiles:
+    - coverage_convert # coverage_convert script
+    - coverage_parse_export # coverage_parse_export script
+    - apollo_archives.yaml # coverage archive export config
+    - ../../src # src source dir
+  outfile_paths:
+    - \./memgraph/tools/apollo/generated/\.coverage_summary
+    - \./memgraph/tools/apollo/generated/\coverage.json

tools/apollo/build_diff

@@ -1 +0,0 @@
-

tools/apollo/coverage_convert

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+project_dir="$( dirname "$( dirname "$script_dir" )" )"
+
+generated_dir="$script_dir/generated"
+html_dir="$generated_dir/html"
+data_file="$generated_dir/default.profdata"
+json_file="$generated_dir/report.json"
+coverage_file="$generated_dir/coverage.json"
+summary_file="$generated_dir/.coverage_summary"
+
+# cleanup output directory
+if [ -d "$generated_dir" ]; then
+    rm -rf "$generated_dir"
+fi
+mkdir "$generated_dir"
+
+# merge raw coverage info
+raw_files="$( find "$HOME" -name "*.profraw" | tr '\n' ' ' )"
+llvm-profdata-5.0 merge -sparse $raw_files -o "$data_file"
+
+# create list of binaries
+cnt=0
+obj_files=""
+for prof_file in $raw_files; do
+    if [ $cnt -gt 0 ]; then
+        obj_files+=" -object "
+    fi
+    obj_files+="$( find "$( dirname "$prof_file" )" -executable -type f | head -n 1 )"
+    cnt=$((cnt + 1))
+done
+
+# create list of source files
+src_files=$( find "$HOME" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' )
+
+# generate html output
+llvm-cov-5.0 show $obj_files \
+    -format html \
+    -instr-profile "$data_file" \
+    -o "$html_dir" \
+    -show-line-counts-or-regions \
+    -Xdemangler c++filt -Xdemangler -n \
+    $src_files
+
+# fix names in html output
+coverage_dir="$html_dir/coverage"
+mv $coverage_dir/workspace/memgraph/* $html_dir/coverage/
+rm -r $coverage_dir/workspace
+find $coverage_dir -name "*.html" -exec sed -i 's@/workspace/memgraph/@@g' {} \;
+find $coverage_dir -name "*.html" -exec sed -i 's@../../style.css@style.css@g' {} \;
+sed -i 's@/workspace/memgraph@@g' $html_dir/index.html
+
+# generate json output
+llvm-cov-5.0 export $obj_files \
+    -instr-profile "$data_file" \
+    -Xdemangler c++filt -Xdemangler -n \
+    $src_files > "$json_file"
+
+# process json output
+$script_dir/coverage_parse_export "$json_file" "$coverage_file" "$summary_file" $src_files

tools/apollo/coverage_parse_export

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import sys
+from collections import defaultdict
+
+def lines2phabricator(filename, lines):
+    ret = ""
+    numlines = 0
+    with open(filename) as f:
+        for row in f:
+            numlines += 1
+    for i in range(1, numlines + 1):
+        val = lines[i]
+        if val is None: ret += "N"
+        elif val == 0: ret += "U"
+        else: ret += "C"
+    return ret
+
+parser = argparse.ArgumentParser(description='Parse llvm-cov export data.')
+parser.add_argument('input', help='input file')
+parser.add_argument('coverage', help='coverage output file')
+parser.add_argument('summary', help='summary output file')
+parser.add_argument('files', nargs='+', help='files to process')
+args = parser.parse_args()
+
+# for specification of the format see:
+# https://github.com/llvm-mirror/llvm/blob/master/tools/llvm-cov/CoverageExporterJson.cpp
+
+data = json.load(open(args.input, "r"))
+
+totals = defaultdict(lambda: defaultdict(int))
+sources = defaultdict(lambda: defaultdict(lambda: None))
+for export in data["data"]:
+    for cfile in export["files"]:
+        for segment in cfile["segments"]:
+            filename = cfile["filename"]
+            if not filename in args.files: continue
+            line, col, count, has_count, is_region_entry = segment
+            sources[filename][line] = count
+    for function in export["functions"]:
+        for region in function["regions"]:
+            line_start, column_start, line_end, column_end, execution_count, \
+                    file_id, expanded_file_id, kind = region
+            filename = function["filenames"][file_id]
+            if filename not in args.files: continue
+            for i in range(line_start, line_end + 1):
+                sources[filename][i] = execution_count
+    for total, values in export["totals"].items():
+        for key, value in values.items():
+            totals[total][key] += value
+
+coverage = {}
+for filename, lines in sources.items():
+    path = "/".join(filename.split("/")[3:])
+    coverage[path] = lines2phabricator(filename, lines)
+
+with open(args.coverage, "w") as f:
+    json.dump(coverage, f)
+
+summary = """==== Code coverage: ====
+
+<table>
+<tr><th>Coverage</th><th>Total</th></tr>
+"""
+ROW = "<tr><td>{name}</td><td>{covered} / {count} ({percent:.2%})</td></tr>\n"
+for what in ["functions", "instantiations", "lines", "regions"]:
+    now = totals[what]
+    now["percent"] = now["covered"] / now["count"]
+    summary += ROW.format(**dict(totals[what], name=what.capitalize()))
+summary += "</table>\n"
+
+with open(args.summary, "w") as f:
+    f.write(summary)