tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add single node memgraph binary

Browse Source 
Summary:
This binary is installed and packaged for release. This is just a quick
solution for releasing the Community 0.10 version. We still need to
setup the installation and packaging for both the Enterprise and
Community versions. Additionally, the automated build system needs to
test both binaries for correct behaviour. Obviously, some tests can only
be run on one of the 2 versions.

Reviewers: mferencevic, buda

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1363
This commit is contained in:
Teon Banek 2018-04-20 14:58:49 +02:00
parent c76170a9db
commit 939056eac7
7 changed files with 174 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -186,6 +186,7 @@ option(EXPERIMENTAL "Build experimental binaries" OFF)
option(CUSTOMERS "Build customer binaries" OFF)
option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
option(TOOLS "Build tools binaries" ON)
option(MG_COMMUNITY "Build Memgraph Community Edition" OFF)
if (TEST_COVERAGE)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@ -196,6 +197,10 @@ if (TEST_COVERAGE)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
endif()
if (MG_COMMUNITY)
add_definitions(-DMG_COMMUNITY)
endif()
# Add subprojects
include_directories(src)
add_subdirectory(src)

4
libs/CMakeLists.txt
View File

@ -11,6 +11,10 @@ function(import_header_library name include_dir)
add_library(${name} INTERFACE IMPORTED GLOBAL)
set_property(TARGET ${name} PROPERTY
INTERFACE_INCLUDE_DIRECTORIES ${include_dir})
string(TOUPPER ${name} _upper_name)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE)
mark_as_advanced(${_upper_name}_INCLUDE_DIR)
endfunction(import_header_library)
function(import_library name type location)

8
release/examples/build_examples
View File

@ -1,21 +1,21 @@
#!/bin/bash -e
function print_help () {
echo "Usage: $0 MEMGRAPH_BUILD_DIR"
echo "Usage: $0 MEMGRAPH_EXE BOLT_CLIENT_EXE"
echo "Build example snapshots using the compiled memgraph."
}
if [[ $# -ne 1 ]]; then
if [[ $# -ne 2 ]]; then
print_help
exit 1
fi
memgraph_exe="$1/memgraph"
memgraph_exe="$1"
if [[ ! -x ${memgraph_exe} ]]; then
echo "Expected memgraph executable at '${memgraph_exe}'"
exit 1
fi
bolt_client_exe="$1/tests/manual/bolt_client"
bolt_client_exe="$2"
if [[ ! -x ${bolt_client_exe} ]]; then
echo "Expected bolt_client executable at '${bolt_client_exe}'"
exit 1

6
src/CMakeLists.txt
View File

@ -153,6 +153,10 @@ install(FILES ${CMAKE_SOURCE_DIR}/release/memgraph.service
# Install examples
set(examples ${CMAKE_SOURCE_DIR}/release/examples)
install(CODE "execute_process(COMMAND ${examples}/build_examples ${CMAKE_BINARY_DIR}
install(
CODE
"execute_process(COMMAND ${examples}/build_examples
${CMAKE_CURRENT_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/tests/manual/bolt_client
WORKING_DIRECTORY ${examples})")
install(DIRECTORY ${examples}/build/ DESTINATION share/memgraph/examples)

14
src/database/config.cpp
View File

@ -27,6 +27,7 @@ DEFINE_int32(gc_cycle_sec, 30,
"Amount of time between starts of two cleaning cycles in seconds. "
"-1 to turn off.");
#ifndef MG_COMMUNITY
// Distributed master/worker flags.
DEFINE_VALIDATED_HIDDEN_int32(worker_id, 0,
"ID of a worker in a distributed system. Igored "
@ -53,7 +54,9 @@ DEFINE_VALIDATED_HIDDEN_int32(rpc_num_workers,
std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers (RPC)",
FLAG_IN_RANGE(1, INT32_MAX));
#endif
// clang-format off
database::Config::Config()
// Durability flags.
: durability_enabled{FLAGS_durability_enabled},
@ -64,11 +67,16 @@ database::Config::Config()
snapshot_on_exit{FLAGS_snapshot_on_exit},
// Misc flags.
gc_cycle_sec{FLAGS_gc_cycle_sec},
query_execution_time_sec{FLAGS_query_execution_time_sec},
rpc_num_workers{FLAGS_rpc_num_workers},
query_execution_time_sec{FLAGS_query_execution_time_sec}
#ifndef MG_COMMUNITY
,
// Distributed flags.
rpc_num_workers{FLAGS_rpc_num_workers},
worker_id{FLAGS_worker_id},
master_endpoint{FLAGS_master_host,
static_cast<uint16_t>(FLAGS_master_port)},
worker_endpoint{FLAGS_worker_host,
static_cast<uint16_t>(FLAGS_worker_port)} {}
static_cast<uint16_t>(FLAGS_worker_port)}
#endif
{}
// clang-format on

8
src/database/graph_db.hpp
View File

@ -43,12 +43,12 @@ struct Config {
// Misc flags.
int gc_cycle_sec;
int query_execution_time_sec;
int rpc_num_workers;
// Distributed master/worker flags.
int worker_id;
io::network::Endpoint master_endpoint;
io::network::Endpoint worker_endpoint;
int rpc_num_workers{0};
int worker_id{0};
io::network::Endpoint master_endpoint{"0.0.0.0", 0};
io::network::Endpoint worker_endpoint{"0.0.0.0", 0};
};
/**

208
src/memgraph_bolt.cpp
View File

@ -1,28 +1,28 @@
#include <algorithm>
#include <chrono>
#include <csignal>
#include <experimental/filesystem>
#include <iostream>
#include <cstdint>
#include <exception>
#include <functional>
#include <limits>
#include <string>
#include <thread>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include "communication/bolt/v1/session.hpp"
#include "communication/server.hpp"
#include "config.hpp"
#include "database/graph_db.hpp"
#include "distributed/coordination_master.hpp"
#include "distributed/coordination_worker.hpp"
#include "io/network/endpoint.hpp"
#include "stats/stats.hpp"
#include "utils/flag_validation.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/scheduler.hpp"
#include "utils/signals.hpp"
#include "utils/stacktrace.hpp"
#include "utils/sysinfo/memory.hpp"
#include "utils/terminate_handler.hpp"
#include "version.hpp"
namespace fs = std::experimental::filesystem;
// Common stuff for enterprise and community editions
using communication::bolt::SessionData;
using SessionT = communication::bolt::Session<communication::InputStream,
communication::OutputStream>;
@ -49,15 +49,6 @@ DEFINE_uint64(memory_warning_threshold, 1024,
"less available RAM it will log a warning. Set to 0 to "
"disable.");
// Distributed flags.
DEFINE_HIDDEN_bool(
master, false,
"If this Memgraph server is the master in a distributed deployment.");
DEFINE_HIDDEN_bool(
worker, false,
"If this Memgraph server is a worker in a distributed deployment.");
DECLARE_int32(worker_id);
// Needed to correctly handle memgraph destruction from a signal handler.
// Without having some sort of a flag, it is possible that a signal is handled
// when we are exiting main, inside destructors of database::GraphDb and
@ -65,9 +56,11 @@ DECLARE_int32(worker_id);
// which is in half destructed state, causing invalid memory access and crash.
volatile sig_atomic_t is_shutting_down = 0;
// Registers the given shutdown function with the appropriate signal handlers.
// See implementation for details.
void InitSignalHandlers(const std::function<void()> &shutdown) {
/// Set up signal handlers and register `shutdown` on SIGTERM and SIGINT.
/// In most cases you don't have to call this. If you are using a custom server
/// startup function for `WithInit`, then you probably need to use this to
/// shutdown your server.
void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
// Prevent handling shutdown inside a shutdown. For example, SIGINT handler
// being interrupted by SIGTERM before is_shutting_down is set, thus causing
// double shutdown.
@ -76,6 +69,13 @@ void InitSignalHandlers(const std::function<void()> &shutdown) {
sigaddset(&block_shutdown_signals, SIGTERM);
sigaddset(&block_shutdown_signals, SIGINT);
// Wrap the shutdown function in a safe way to prevent recursive shutdown.
auto shutdown = [shutdown_fun]() {
if (is_shutting_down) return;
is_shutting_down = 1;
shutdown_fun();
};
CHECK(utils::SignalHandler::RegisterHandler(utils::Signal::Terminate,
shutdown, block_shutdown_signals))
<< "Unable to register SIGTERM handler!";
@ -90,6 +90,103 @@ void InitSignalHandlers(const std::function<void()> &shutdown) {
})) << "Unable to register SIGUSR1 handler!";
}
/// Run the Memgraph server.
///
/// Sets up all the required state before running `memgraph_main` and does any
/// required cleanup afterwards. `get_stats_prefix` is used to obtain the
/// prefix when logging Memgraph's statistics.
///
/// Command line arguments and configuration files are read before calling any
/// of the supplied functions. Therefore, you should use flags only from those
/// functions, and *not before* invoking `WithInit`.
///
/// This should be the first and last thing a OS specific main function does.
///
/// A common example of usage is:
///
/// @code
/// int main(int argc, char *argv[]) {
/// auto get_stats_prefix = []() -> std::string { return "memgraph"; };
/// return WithInit(argc, argv, get_stats_prefix, SingleNodeMain);
/// }
/// @endcode
///
/// If you wish to start Memgraph server in another way, you can pass a
/// `memgraph_main` functions which does that. You should take care to call
/// `InitSignalHandlers` with appropriate function to shutdown the server you
/// started.
int WithInit(int argc, char **argv,
const std::function<std::string()> &get_stats_prefix,
const std::function<void()> &memgraph_main) {
gflags::SetVersionString(version_string);
// Load config before parsing arguments, so that flags from the command line
// overwrite the config.
LoadConfig();
gflags::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
google::SetLogDestination(google::INFO, FLAGS_log_file.c_str());
google::SetLogSymlink(google::INFO, FLAGS_log_link_basename.c_str());
// Unhandled exception handler init.
std::set_terminate(&utils::TerminateHandler);
stats::InitStatsLogging(get_stats_prefix());
utils::OnScopeExit stop_stats([] { stats::StopStatsLogging(); });
// Start memory warning logger.
utils::Scheduler mem_log_scheduler;
if (FLAGS_memory_warning_threshold > 0) {
mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] {
auto free_ram_mb = utils::sysinfo::AvailableMem() / 1024;
if (free_ram_mb < FLAGS_memory_warning_threshold)
LOG(WARNING) << "Running out of available RAM, only " << free_ram_mb
<< " MB left.";
});
}
memgraph_main();
return 0;
}
void SingleNodeMain() {
google::SetUsageMessage("Memgraph single-node database server");
database::SingleNode db;
SessionData session_data{db};
ServerT server({FLAGS_interface, static_cast<uint16_t>(FLAGS_port)},
session_data, FLAGS_session_inactivity_timeout, "Bolt",
FLAGS_num_workers);
// Handler for regular termination signals
auto shutdown = [&server] {
// Server needs to be shutdown first and then the database. This prevents a
// race condition when a transaction is accepted during server shutdown.
server.Shutdown();
};
InitSignalHandlers(shutdown);
server.AwaitShutdown();
}
// End common stuff for enterprise and community editions
#ifdef MG_COMMUNITY
int main(int argc, char **argv) {
return WithInit(argc, argv, []() { return "memgraph"; }, SingleNodeMain);
}
#else // enterprise edition
// Distributed flags.
DEFINE_HIDDEN_bool(
master, false,
"If this Memgraph server is the master in a distributed deployment.");
DEFINE_HIDDEN_bool(
worker, false,
"If this Memgraph server is a worker in a distributed deployment.");
DECLARE_int32(worker_id);
void MasterMain() {
google::SetUsageMessage("Memgraph distributed master");
@ -101,8 +198,6 @@ void MasterMain() {
// Handler for regular termination signals
auto shutdown = [&server] {
if (is_shutting_down) return;
is_shutting_down = 1;
// Server needs to be shutdown first and then the database. This prevents a
// race condition when a transaction is accepted during server shutdown.
server.Shutdown();
@ -118,64 +213,17 @@ void WorkerMain() {
db.WaitForShutdown();
}
void SingleNodeMain() {
google::SetUsageMessage("Memgraph single-node database server");
database::SingleNode db;
SessionData session_data{db};
ServerT server({FLAGS_interface, static_cast<uint16_t>(FLAGS_port)},
session_data, FLAGS_session_inactivity_timeout, "Bolt",
FLAGS_num_workers);
// Handler for regular termination signals
auto shutdown = [&server] {
if (is_shutting_down) return;
is_shutting_down = 1;
// Server needs to be shutdown first and then the database. This prevents a
// race condition when a transaction is accepted during server shutdown.
server.Shutdown();
};
InitSignalHandlers(shutdown);
server.AwaitShutdown();
}
int main(int argc, char **argv) {
gflags::SetVersionString(version_string);
// Load config before parsing arguments, so that flags from the command line
// overwrite the config.
LoadConfig();
gflags::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
google::SetLogDestination(google::INFO, FLAGS_log_file.c_str());
google::SetLogSymlink(google::INFO, FLAGS_log_link_basename.c_str());
// Unhandled exception handler init.
std::set_terminate(&utils::TerminateHandler);
std::string stats_prefix;
auto get_stats_prefix = [&]() -> std::string {
if (FLAGS_master) {
stats_prefix = "master";
return "master";
} else if (FLAGS_worker) {
stats_prefix = fmt::format("worker-{}", FLAGS_worker_id);
} else {
stats_prefix = "memgraph";
}
stats::InitStatsLogging(stats_prefix);
utils::OnScopeExit stop_stats([] { stats::StopStatsLogging(); });
// Start memory warning logger.
utils::Scheduler mem_log_scheduler;
if (FLAGS_memory_warning_threshold > 0) {
mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] {
auto free_ram_mb = utils::sysinfo::AvailableMem() / 1024;
if (free_ram_mb < FLAGS_memory_warning_threshold)
LOG(WARNING) << "Running out of available RAM, only " << free_ram_mb
<< " MB left.";
});
return fmt::format("worker-{}", FLAGS_worker_id);
}
return "memgraph";
};
auto memgraph_main = [&]() {
CHECK(!(FLAGS_master && FLAGS_worker))
<< "Can't run Memgraph as worker and master at the same time";
if (FLAGS_master)
@ -184,5 +232,9 @@ int main(int argc, char **argv) {
WorkerMain();
else
SingleNodeMain();
return 0;
};
return WithInit(argc, argv, get_stats_prefix, memgraph_main);
}
#endif // enterprise edition