Extract utils into mg-utils static library and explicitly list tests

Summary:
Utils source files are now moved to a standalone mg-utils library.

Unit and manual tests are no longer collected using glob recursion in
cmake, but are explicitly listed. This allows us to set only required
dependencies of those tests.

Both of these changes should improve compilation and link times, as well
as lower the disk usage.

Additional improvement would be to cleanup utils header files to be
split in .hpp and .cpp as well as merging threading into utils. Other
potential library extractions that shouldn't be difficult are:

  * data_structures
  * io/network
  * communication

Reviewers: buda, mferencevic, dgleich, ipaljak, mculinovic, mtomic, msantl

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1408
This commit is contained in:
Teon Banek 2018-05-29 11:13:13 +02:00
parent 1875be1e34
commit ce29517998
19 changed files with 371 additions and 73 deletions

View File

@ -1,5 +1,8 @@
# CMake configuration for the main memgraph library and executable
# add memgraph sub libraries
add_subdirectory(utils)
# all memgraph src files
set(memgraph_src_files
communication/buffer.cpp
@ -40,6 +43,7 @@ set(memgraph_src_files
io/network/addrinfo.cpp
io/network/endpoint.cpp
io/network/socket.cpp
io/network/utils.cpp
query/common.cpp
query/repl.cpp
query/frontend/ast/ast.cpp
@ -71,11 +75,6 @@ set(memgraph_src_files
transactions/engine_master.cpp
transactions/engine_single_node.cpp
transactions/engine_worker.cpp
utils/demangle.cpp
utils/file.cpp
utils/network.cpp
utils/signals.cpp
utils/watchdog.cpp
)
# -----------------------------------------------------------------------------
@ -147,15 +146,14 @@ endif()
# STATIC library used by memgraph executables
add_library(memgraph_lib STATIC ${memgraph_src_files})
target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS})
target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS} mg-utils)
add_dependencies(memgraph_lib generate_opencypher_parser)
add_dependencies(memgraph_lib generate_lcp)
add_dependencies(memgraph_lib generate_capnp)
# STATIC library used to store key-value pairs
# TODO: Create a utils lib to link with, and remove utils/file.cpp.
add_library(kvstore_lib STATIC storage/kvstore.cpp utils/file.cpp)
target_link_libraries(kvstore_lib stdc++fs fmt gflags glog rocksdb bzip2 zlib)
add_library(kvstore_lib STATIC storage/kvstore.cpp)
target_link_libraries(kvstore_lib stdc++fs mg-utils rocksdb bzip2 zlib)
# Generate a version.hpp file
set(VERSION_STRING ${memgraph_VERSION})

View File

@ -6,7 +6,7 @@
#include "communication/rpc/client.hpp"
#include "distributed/coordination_master.hpp"
#include "distributed/coordination_rpc_messages.hpp"
#include "utils/network.hpp"
#include "io/network/utils.hpp"
namespace distributed {
@ -66,7 +66,7 @@ MasterCoordination::~MasterCoordination() {
for (const auto &kv : workers) {
// Skip master (self).
if (kv.first == 0) continue;
while (utils::CanEstablishConnection(kv.second))
while (io::network::CanEstablishConnection(kv.second))
std::this_thread::sleep_for(0.5s);
}
}

View File

@ -1,4 +1,4 @@
#include "utils/network.hpp"
#include "io/network/utils.hpp"
#include <arpa/inet.h>
#include <netdb.h>
@ -12,7 +12,7 @@
#include "io/network/socket.hpp"
namespace utils {
namespace io::network {
/// Resolves hostname to ip, if already an ip, just returns it
std::string ResolveHostname(std::string hostname) {
@ -58,4 +58,4 @@ bool CanEstablishConnection(const io::network::Endpoint &endpoint) {
return client.Connect(endpoint);
}
}; // namespace utils
}; // namespace io::network

View File

@ -5,7 +5,7 @@
#include "io/network/endpoint.hpp"
namespace utils {
namespace io::network {
/// Resolves hostname to ip, if already an ip, just returns it
std::string ResolveHostname(std::string hostname);
@ -14,6 +14,6 @@ std::string ResolveHostname(std::string hostname);
std::experimental::optional<std::string> GetHostname();
// Try to establish a connection to a remote host
bool CanEstablishConnection(const io::network::Endpoint &endpoint);
bool CanEstablishConnection(const Endpoint &endpoint);
} // namespace utils
} // namespace io::network

8
src/utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
set(utils_src_files
demangle.cpp
file.cpp
signals.cpp
watchdog.cpp)
add_library(mg-utils STATIC ${utils_src_files})
target_link_libraries(mg-utils stdc++fs Threads::Threads fmt glog gflags)

View File

@ -8,8 +8,8 @@
#include "communication/rpc/client.hpp"
#include "io/network/endpoint.hpp"
#include "io/network/utils.hpp"
#include "messages.hpp"
#include "utils/network.hpp"
using namespace communication::rpc;
using namespace std::literals::chrono_literals;
@ -26,7 +26,7 @@ int main(int argc, char **argv) {
// Initialize client.
Client client(io::network::Endpoint(
utils::ResolveHostname(FLAGS_server_interface), FLAGS_server_port));
io::network::ResolveHostname(FLAGS_server_interface), FLAGS_server_port));
// Try to send 100 values to server.
// If requests timeout, try to resend it.

View File

@ -4,7 +4,6 @@
#include "stats/metrics.hpp"
#include "stats/stats.hpp"
#include "utils/network.hpp"
#include "utils/timer.hpp"
#include "common.hpp"

View File

@ -13,9 +13,9 @@
#include <glog/logging.h>
#include <json/json.hpp>
#include "io/network/utils.hpp"
#include "threading/sync/spinlock.hpp"
#include "utils/algorithm.hpp"
#include "utils/network.hpp"
#include "utils/timer.hpp"
#include "long_running_common.hpp"
@ -276,7 +276,7 @@ int main(int argc, char **argv) {
std::cin >> config;
auto independent_nodes_ids = [&] {
Endpoint endpoint(utils::ResolveHostname(FLAGS_address), FLAGS_port);
Endpoint endpoint(io::network::ResolveHostname(FLAGS_address), FLAGS_port);
Client client;
if (!client.Connect(endpoint, FLAGS_username, FLAGS_password)) {
LOG(FATAL) << "Couldn't connect to " << endpoint;

View File

@ -1,27 +1,75 @@
# set current directory name as a test type
get_filename_component(test_type ${CMAKE_CURRENT_SOURCE_DIR} NAME)
set(test_prefix memgraph__manual__)
# get all cpp abs file names recursively starting from current directory
file(GLOB_RECURSE test_type_cpps *.cpp)
message(STATUS "Available ${test_type} cpp files are: ${test_type_cpps}")
add_custom_target(memgraph__manual)
# for each cpp file build binary and register test
foreach(test_cpp ${test_type_cpps})
function(add_manual_test test_cpp)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
add_dependencies(memgraph__manual ${target_name})
endfunction(add_manual_test)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
add_manual_test(antlr_parser.cpp)
target_link_libraries(${test_prefix}antlr_parser antlr_opencypher_parser_lib)
set(target_name memgraph__${test_type}__${exec_name})
add_manual_test(antlr_sigsegv.cpp)
target_link_libraries(${test_prefix}antlr_sigsegv gtest gtest_main
antlr_opencypher_parser_lib mg-utils)
# build exec file
add_executable(${target_name} ${test_cpp})
add_manual_test(antlr_tree_pretty_print.cpp)
target_link_libraries(${test_prefix}antlr_tree_pretty_print antlr_opencypher_parser_lib)
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
add_manual_test(binomial.cpp)
target_link_libraries(${test_prefix}binomial mg-utils)
# link libraries
target_link_libraries(${target_name} memgraph_lib gtest gtest_main
glog kvstore_lib)
endforeach()
add_manual_test(bolt_client.cpp)
target_link_libraries(${test_prefix}bolt_client memgraph_lib)
add_manual_test(card_fraud_generate_snapshot.cpp)
target_link_libraries(${test_prefix}card_fraud_generate_snapshot memgraph_lib)
add_manual_test(card_fraud_local.cpp)
target_link_libraries(${test_prefix}card_fraud_local memgraph_lib)
add_manual_test(distributed_repl.cpp)
target_link_libraries(${test_prefix}distributed_repl memgraph_lib)
add_manual_test(endinan.cpp)
add_manual_test(generate_snapshot.cpp)
target_link_libraries(${test_prefix}generate_snapshot memgraph_lib)
add_manual_test(graph_500_generate_snapshot.cpp)
target_link_libraries(${test_prefix}graph_500_generate_snapshot memgraph_lib)
add_manual_test(kvstore.cpp)
target_link_libraries(${test_prefix}kvstore gtest gtest_main memgraph_lib kvstore_lib)
add_manual_test(query_hash.cpp)
target_link_libraries(${test_prefix}query_hash memgraph_lib)
add_manual_test(query_planner.cpp)
target_link_libraries(${test_prefix}query_planner memgraph_lib)
add_manual_test(raft_rpc.cpp)
target_link_libraries(${test_prefix}raft_rpc memgraph_lib)
add_manual_test(repl.cpp)
target_link_libraries(${test_prefix}repl memgraph_lib)
add_manual_test(single_query.cpp)
target_link_libraries(${test_prefix}single_query memgraph_lib)
add_manual_test(sl_position_and_count.cpp)
target_link_libraries(${test_prefix}sl_position_and_count memgraph_lib)
add_manual_test(stripped_timing.cpp)
target_link_libraries(${test_prefix}stripped_timing memgraph_lib)
add_manual_test(xorshift.cpp)
target_link_libraries(${test_prefix}xorshift mg-utils)

View File

@ -3,7 +3,7 @@
#include "communication/bolt/client.hpp"
#include "io/network/endpoint.hpp"
#include "utils/network.hpp"
#include "io/network/utils.hpp"
#include "utils/timer.hpp"
DEFINE_string(address, "127.0.0.1", "Server address");
@ -16,7 +16,7 @@ int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
// TODO: handle endpoint exception
io::network::Endpoint endpoint(utils::ResolveHostname(FLAGS_address),
io::network::Endpoint endpoint(io::network::ResolveHostname(FLAGS_address),
FLAGS_port);
communication::bolt::Client client;

View File

@ -1,38 +1,283 @@
# set current directory name as a test type
get_filename_component(test_type ${CMAKE_CURRENT_SOURCE_DIR} NAME)
# get all cpp abs file names recursively starting from current directory
file(GLOB_RECURSE test_type_cpps *.cpp)
message(STATUS "Available ${test_type} cpp files are: ${test_type_cpps}")
set(test_prefix memgraph__unit__)
add_custom_target(memgraph__unit)
# for each cpp file build binary and register test
foreach(test_cpp ${test_type_cpps})
function(add_unit_test test_cpp)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
target_link_libraries(${target_name} gtest gmock gtest_main)
# register test
set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)
add_test(${target_name} ${exec_name} --gtest_output=xml:${output_path})
# add to memgraph__unit target
add_dependencies(memgraph__unit ${target_name})
endfunction(add_unit_test)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
add_unit_test(bolt_chunked_decoder_buffer.cpp)
target_link_libraries(${test_prefix}bolt_chunked_decoder_buffer memgraph_lib)
set(target_name memgraph__${test_type}__${exec_name})
add_unit_test(bolt_chunked_encoder_buffer.cpp)
target_link_libraries(${test_prefix}bolt_chunked_encoder_buffer memgraph_lib)
# build exec file
add_executable(${target_name} ${test_cpp})
add_unit_test(bolt_decoder.cpp)
target_link_libraries(${test_prefix}bolt_decoder memgraph_lib)
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
add_unit_test(bolt_encoder.cpp)
target_link_libraries(${test_prefix}bolt_encoder memgraph_lib)
# link libraries
target_link_libraries(${target_name} memgraph_lib)
# gtest
target_link_libraries(${target_name} gtest gmock gtest_main)
add_unit_test(bolt_result_stream.cpp)
target_link_libraries(${test_prefix}bolt_result_stream memgraph_lib)
# register test
set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)
add_test(${target_name} ${exec_name} --gtest_output=xml:${output_path})
add_unit_test(bolt_session.cpp)
target_link_libraries(${test_prefix}bolt_session memgraph_lib)
# add to memgraph__unit target
add_dependencies(memgraph__unit ${target_name})
add_unit_test(communication_buffer.cpp)
target_link_libraries(${test_prefix}communication_buffer memgraph_lib)
endforeach()
add_unit_test(concurrent_id_mapper_distributed.cpp)
target_link_libraries(${test_prefix}concurrent_id_mapper_distributed memgraph_lib)
add_unit_test(concurrent_id_mapper_single_node.cpp)
target_link_libraries(${test_prefix}concurrent_id_mapper_single_node memgraph_lib)
add_unit_test(concurrent_map_access.cpp)
target_link_libraries(${test_prefix}concurrent_map_access memgraph_lib)
add_unit_test(concurrent_map.cpp)
target_link_libraries(${test_prefix}concurrent_map memgraph_lib)
add_unit_test(counters.cpp)
target_link_libraries(${test_prefix}counters memgraph_lib)
add_unit_test(cypher_main_visitor.cpp)
target_link_libraries(${test_prefix}cypher_main_visitor memgraph_lib)
add_unit_test(database_key_index.cpp)
target_link_libraries(${test_prefix}database_key_index memgraph_lib)
add_unit_test(database_label_property_index.cpp)
target_link_libraries(${test_prefix}database_label_property_index memgraph_lib)
add_unit_test(database_master.cpp)
target_link_libraries(${test_prefix}database_master memgraph_lib)
add_unit_test(database_transaction_timeout.cpp)
target_link_libraries(${test_prefix}database_transaction_timeout memgraph_lib)
add_unit_test(datastructure_union_find.cpp)
target_link_libraries(${test_prefix}datastructure_union_find memgraph_lib)
add_unit_test(deferred_deleter.cpp)
target_link_libraries(${test_prefix}deferred_deleter memgraph_lib)
add_unit_test(distributed_bfs.cpp)
target_link_libraries(${test_prefix}distributed_bfs memgraph_lib)
add_unit_test(distributed_coordination.cpp)
target_link_libraries(${test_prefix}distributed_coordination memgraph_lib)
add_unit_test(distributed_data_exchange.cpp)
target_link_libraries(${test_prefix}distributed_data_exchange memgraph_lib)
add_unit_test(distributed_durability.cpp)
target_link_libraries(${test_prefix}distributed_durability memgraph_lib)
add_unit_test(distributed_gc.cpp)
target_link_libraries(${test_prefix}distributed_gc memgraph_lib)
add_unit_test(distributed_graph_db.cpp)
target_link_libraries(${test_prefix}distributed_graph_db memgraph_lib)
add_unit_test(distributed_interpretation.cpp)
target_link_libraries(${test_prefix}distributed_interpretation memgraph_lib)
add_unit_test(distributed_query_plan.cpp)
target_link_libraries(${test_prefix}distributed_query_plan memgraph_lib)
add_unit_test(distributed_serialization.cpp)
target_link_libraries(${test_prefix}distributed_serialization memgraph_lib)
add_unit_test(distributed_updates.cpp)
target_link_libraries(${test_prefix}distributed_updates memgraph_lib)
add_unit_test(durability.cpp)
target_link_libraries(${test_prefix}durability memgraph_lib)
add_unit_test(dynamic_bitset.cpp)
target_link_libraries(${test_prefix}dynamic_bitset memgraph_lib)
add_unit_test(gid.cpp)
target_link_libraries(${test_prefix}gid memgraph_lib)
add_unit_test(graph_db_accessor.cpp)
target_link_libraries(${test_prefix}graph_db_accessor memgraph_lib)
add_unit_test(graph_db_accessor_index_api.cpp)
target_link_libraries(${test_prefix}graph_db_accessor_index_api memgraph_lib)
add_unit_test(graph_db.cpp)
target_link_libraries(${test_prefix}graph_db memgraph_lib)
add_unit_test(interpreter.cpp)
target_link_libraries(${test_prefix}interpreter memgraph_lib)
add_unit_test(metrics.cpp)
target_link_libraries(${test_prefix}metrics memgraph_lib)
add_unit_test(mvcc.cpp)
target_link_libraries(${test_prefix}mvcc memgraph_lib)
add_unit_test(mvcc_find.cpp)
target_link_libraries(${test_prefix}mvcc_find memgraph_lib)
add_unit_test(mvcc_gc.cpp)
target_link_libraries(${test_prefix}mvcc_gc memgraph_lib)
add_unit_test(mvcc_one_transaction.cpp)
target_link_libraries(${test_prefix}mvcc_one_transaction memgraph_lib)
add_unit_test(mvcc_parallel_update.cpp)
target_link_libraries(${test_prefix}mvcc_parallel_update memgraph_lib)
add_unit_test(network_endpoint.cpp)
target_link_libraries(${test_prefix}network_endpoint memgraph_lib)
add_unit_test(network_timeouts.cpp)
target_link_libraries(${test_prefix}network_timeouts memgraph_lib)
add_unit_test(network_utils.cpp)
target_link_libraries(${test_prefix}network_utils memgraph_lib)
add_unit_test(property_value_store.cpp)
target_link_libraries(${test_prefix}property_value_store memgraph_lib)
add_unit_test(query_cost_estimator.cpp)
target_link_libraries(${test_prefix}query_cost_estimator memgraph_lib)
add_unit_test(query_expression_evaluator.cpp)
target_link_libraries(${test_prefix}query_expression_evaluator memgraph_lib)
add_unit_test(query_plan_accumulate_aggregate.cpp)
target_link_libraries(${test_prefix}query_plan_accumulate_aggregate memgraph_lib)
add_unit_test(query_plan_bag_semantics.cpp)
target_link_libraries(${test_prefix}query_plan_bag_semantics memgraph_lib)
add_unit_test(query_plan_create_set_remove_delete.cpp)
target_link_libraries(${test_prefix}query_plan_create_set_remove_delete memgraph_lib)
add_unit_test(query_plan_edge_cases.cpp)
target_link_libraries(${test_prefix}query_plan_edge_cases memgraph_lib)
add_unit_test(query_plan_match_filter_return.cpp)
target_link_libraries(${test_prefix}query_plan_match_filter_return memgraph_lib)
add_unit_test(query_planner.cpp)
target_link_libraries(${test_prefix}query_planner memgraph_lib)
add_unit_test(query_semantic.cpp)
target_link_libraries(${test_prefix}query_semantic memgraph_lib)
add_unit_test(query_variable_start_planner.cpp)
target_link_libraries(${test_prefix}query_variable_start_planner memgraph_lib)
add_unit_test(queue.cpp)
target_link_libraries(${test_prefix}queue memgraph_lib)
add_unit_test(raft.cpp)
target_link_libraries(${test_prefix}raft memgraph_lib)
add_unit_test(raft_storage.cpp)
target_link_libraries(${test_prefix}raft_storage memgraph_lib)
add_unit_test(record_edge_vertex_accessor.cpp)
target_link_libraries(${test_prefix}record_edge_vertex_accessor memgraph_lib)
add_unit_test(ring_buffer.cpp)
target_link_libraries(${test_prefix}ring_buffer memgraph_lib)
add_unit_test(rpc.cpp)
target_link_libraries(${test_prefix}rpc memgraph_lib)
add_unit_test(rpc_worker_clients.cpp)
target_link_libraries(${test_prefix}rpc_worker_clients memgraph_lib)
add_unit_test(rwlock.cpp)
target_link_libraries(${test_prefix}rwlock memgraph_lib)
add_unit_test(serialization.cpp)
target_link_libraries(${test_prefix}serialization memgraph_lib)
add_unit_test(skiplist_access.cpp)
target_link_libraries(${test_prefix}skiplist_access memgraph_lib)
add_unit_test(skiplist_gc.cpp)
target_link_libraries(${test_prefix}skiplist_gc memgraph_lib)
add_unit_test(skiplist_position_and_count.cpp)
target_link_libraries(${test_prefix}skiplist_position_and_count memgraph_lib)
add_unit_test(skiplist_reverse_iteration.cpp)
target_link_libraries(${test_prefix}skiplist_reverse_iteration memgraph_lib)
add_unit_test(skiplist_suffix.cpp)
target_link_libraries(${test_prefix}skiplist_suffix memgraph_lib)
add_unit_test(state_delta.cpp)
target_link_libraries(${test_prefix}state_delta memgraph_lib)
add_unit_test(static_bitset.cpp)
target_link_libraries(${test_prefix}static_bitset memgraph_lib)
add_unit_test(storage_address.cpp)
target_link_libraries(${test_prefix}storage_address memgraph_lib)
add_unit_test(stripped.cpp)
target_link_libraries(${test_prefix}stripped memgraph_lib)
add_unit_test(thread_pool.cpp)
target_link_libraries(${test_prefix}thread_pool memgraph_lib)
add_unit_test(transaction_engine_distributed.cpp)
target_link_libraries(${test_prefix}transaction_engine_distributed memgraph_lib)
add_unit_test(transaction_engine_single_node.cpp)
target_link_libraries(${test_prefix}transaction_engine_single_node memgraph_lib)
add_unit_test(typed_value.cpp)
target_link_libraries(${test_prefix}typed_value memgraph_lib)
# Test mg-utils
add_unit_test(utils_demangle.cpp)
target_link_libraries(${test_prefix}utils_demangle mg-utils)
add_unit_test(utils_exceptions.cpp)
target_link_libraries(${test_prefix}utils_exceptions mg-utils)
add_unit_test(utils_executor.cpp)
target_link_libraries(${test_prefix}utils_executor mg-utils)
add_unit_test(utils_on_scope_exit.cpp)
target_link_libraries(${test_prefix}utils_on_scope_exit mg-utils)
add_unit_test(utils_scheduler.cpp)
target_link_libraries(${test_prefix}utils_scheduler mg-utils)
add_unit_test(utils_signals.cpp)
target_link_libraries(${test_prefix}utils_signals mg-utils)
add_unit_test(utils_string.cpp)
target_link_libraries(${test_prefix}utils_string mg-utils)
add_unit_test(utils_timestamp.cpp)
target_link_libraries(${test_prefix}utils_timestamp mg-utils)
add_unit_test(utils_watchdog.cpp)
target_link_libraries(${test_prefix}utils_watchdog mg-utils)

View File

@ -1,9 +1,9 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "utils/network.hpp"
#include "io/network/utils.hpp"
using namespace utils;
using namespace io::network;
TEST(ResolveHostname, Simple) {
auto result = ResolveHostname("localhost");