From 283a91cc60738c4d61af6502a8136ff25e0b8b4f Mon Sep 17 00:00:00 2001 From: Teon Banek Date: Thu, 31 Oct 2019 16:36:34 +0100 Subject: [PATCH] Integrate loading openCypher module procedures Summary: All mgp_* symbols are exported from Memgraph executable, no other symbols should be visible. The primary C API header, mg_procedure.h, is now part of the installation. Also, added a shippable query module example. Directory `query_modules` is meant to contain sources of modules we write and ship as part of the installation. Currently, there's only an example module, but there may be potentially more. Some modules could only be installed as part of the enterprise release. For Memgraph to load custom procedures, it needs to be started with a flag pointing to a directory with compiled shared libraries implementing those procedures. Reviewers: mferencevic, ipaljak, llugovic, dsantl, buda Reviewed By: mferencevic Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D2538 --- CMakeLists.txt | 5 +++ include/mg_procedure.syms | 3 ++ query_modules/CMakeLists.txt | 19 +++++++++++ query_modules/example.c | 64 ++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 17 ++++++++++ src/memgraph.cpp | 24 ++++++++++++++ 6 files changed, 132 insertions(+) create mode 100644 include/mg_procedure.syms create mode 100644 query_modules/CMakeLists.txt create mode 100644 query_modules/example.c diff --git a/CMakeLists.txt b/CMakeLists.txt index a23920b38..0e5a66d90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,7 @@ option(EXPERIMENTAL "Build experimental binaries" OFF) option(CUSTOMERS "Build customer binaries" OFF) option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF) option(TOOLS "Build tools binaries" ON) +option(QUERY_MODULES "Build query modules containing custom procedures" ON) option(MG_COMMUNITY "Build Memgraph Community Edition" OFF) option(ASAN "Build with Address Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF) option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF) @@ -296,6 +297,10 @@ if(TOOLS) add_subdirectory(tools) endif() +if(QUERY_MODULES) + add_subdirectory(query_modules) +endif() + # ----------------------------------------------------------------------------- # ---- Setup CPack -------- diff --git a/include/mg_procedure.syms b/include/mg_procedure.syms new file mode 100644 index 000000000..9a3241427 --- /dev/null +++ b/include/mg_procedure.syms @@ -0,0 +1,3 @@ +{ + mgp_*; +}; diff --git a/query_modules/CMakeLists.txt b/query_modules/CMakeLists.txt new file mode 100644 index 000000000..72600ea02 --- /dev/null +++ b/query_modules/CMakeLists.txt @@ -0,0 +1,19 @@ +# Memgraph Query Modules CMake configuration +# You should use the top level CMake configuration with -DQUERY_MODULES=ON +# These modules are meant to be shipped with Memgraph installation. + +project(memgraph_query_modules VERSION ${memgraph_VERSION}) + +disallow_in_source_build() + +# Everything that is installed here, should be under the "query_modules" component. +set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules") + +add_library(example SHARED example.c) +target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/include) +target_compile_options(example PRIVATE -Wall) +install(PROGRAMS $ + DESTINATION lib/memgraph/query_modules + RENAME example.so) +# Also install the source of the example, so user can read it. +install(FILES example.c DESTINATION lib/memgraph/query_modules) diff --git a/query_modules/example.c b/query_modules/example.c new file mode 100644 index 000000000..4d50da66a --- /dev/null +++ b/query_modules/example.c @@ -0,0 +1,64 @@ +// Compile with clang or gcc: +// clang -Wall -shared -fPIC -I example.c -o example.so +// for installed Memgraph will usually be something +// like `/usr/include/memgraph` or `/usr/local/include/memgraph`. +// To use the compiled module, you need to run Memgraph configured to load +// modules from the directory where the compiled module can be found. +#include "mg_procedure.h" + +// This example procedure returns 2 fields: `args` and `result`. +// * `args` is a copy of arguments passed to the procedure. +// * `result` is the result of this procedure, a "Hello World!" string. +// In case of memory errors, this function will report them and finish executing. +// +// The procedure can be invoked in openCypher using the following call: +// CALL example(1, 2, 3) YIELD args, result; +// Naturally, you may pass in different arguments or yield less fields. +void mgp_main(const struct mgp_list *args, const struct mgp_graph *graph, + struct mgp_result *result, struct mgp_memory *memory) { + struct mgp_list *args_copy = mgp_list_make_empty(mgp_list_size(args), memory); + if (args_copy == NULL) goto error_memory; + for (size_t i = 0; i < mgp_list_size(args); ++i) { + int success = mgp_list_append(args_copy, mgp_list_at(args, i)); + if (!success) goto error_free_list; + } + struct mgp_result_record *record = mgp_result_new_record(result); + if (record == NULL) goto error_free_list; + // Transfer ownership of args_copy to mgp_value. + struct mgp_value *args_value = mgp_value_make_list(args_copy); + if (args_value == NULL) goto error_free_list; + int args_inserted = mgp_result_record_insert(record, "args", args_value); + // Release `args_value` and contained `args_copy`. + mgp_value_destroy(args_value); + if (!args_inserted) goto error_memory; + struct mgp_value *hello_world_value = + mgp_value_make_string("Hello World!", memory); + if (hello_world_value == NULL) goto error_memory; + int result_inserted = + mgp_result_record_insert(record, "result", hello_world_value); + mgp_value_destroy(hello_world_value); + if (!result_inserted) goto error_memory; + // We have successfully finished, so return without error reporting. + return; + +error_free_list: + mgp_list_destroy(args_copy); +error_memory: + mgp_result_set_error_msg(result, "Not enough memory!"); + return; +} + +// This is an optional function if you need to initialize any global state when +// your module is loaded. +int mgp_init_module() { + // Return 0 to indicate success. + return 0; +} + +// This is an optional function if you need to release any resources before the +// module is unloaded. You will probably need this if you acquired some +// resources in mgp_init_module. +int mgp_shutdown_module() { + // Return 0 to indicate success. + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5138896bb..25cad88ba 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,6 +105,10 @@ target_link_libraries(mg-single-node ${MG_SINGLE_NODE_LIBS}) add_dependencies(mg-single-node generate_opencypher_parser) add_dependencies(mg-single-node generate_lcp_single_node) target_compile_definitions(mg-single-node PUBLIC MG_SINGLE_NODE) +# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which +# should be dynamically exported, so that `dlopen` can correctly link the +# symbols in custom procedure module libraries. +target_link_libraries(mg-single-node "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms") # ---------------------------------------------------------------------------- # END Memgraph Single Node @@ -161,6 +165,10 @@ add_dependencies(mg-single-node-v2 generate_lcp_common) target_compile_definitions(mg-single-node-v2 PUBLIC MG_SINGLE_NODE_V2) add_executable(memgraph-v2 memgraph.cpp) target_link_libraries(memgraph-v2 mg-single-node-v2 kvstore_lib telemetry_lib) +# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which +# should be dynamically exported, so that `dlopen` can correctly link the +# symbols in custom procedure module libraries. +target_link_libraries(mg-single-node-v2 "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms") # ---------------------------------------------------------------------------- # END Memgraph Single Node v2 @@ -250,6 +258,12 @@ target_link_libraries(mg-single-node-ha ${MG_SINGLE_NODE_HA_LIBS}) add_dependencies(mg-single-node-ha generate_opencypher_parser) add_dependencies(mg-single-node-ha generate_lcp_single_node_ha) target_compile_definitions(mg-single-node-ha PUBLIC MG_SINGLE_NODE_HA) +# TODO: Make these symbols visible once we add support for custom procedure +# modules in HA. +# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which +# should be dynamically exported, so that `dlopen` can correctly link the +# symbols in custom procedure module libraries. +# target_link_libraries(mg-single-node-ha "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms") # ---------------------------------------------------------------------------- # END Memgraph Single Node High Availability @@ -306,6 +320,9 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph") # we cannot use the recommended `install(TARGETS ...)`. install(PROGRAMS $ DESTINATION lib/memgraph RENAME memgraph) +# Install the include file for writing custom procedures. +install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h + DESTINATION include/memgraph) # Install the config file (must use absolute path). install(FILES ${CMAKE_SOURCE_DIR}/config/community.conf DESTINATION /etc/memgraph RENAME memgraph.conf) diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 1a699766b..3329133c1 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -18,8 +18,11 @@ #include "memgraph_init.hpp" #include "query/exceptions.hpp" #include "telemetry/telemetry.hpp" +#include "utils/file.hpp" #include "utils/flag_validation.hpp" +#include "query/procedure/module.hpp" + // General purpose flags. DEFINE_string(interface, "0.0.0.0", "Communication interface on which to listen."); @@ -88,6 +91,16 @@ DEFINE_VALIDATED_int32( "Interval (in milliseconds) used for flushing the audit log buffer.", FLAG_IN_RANGE(10, INT32_MAX)); +DEFINE_VALIDATED_string( + query_modules_directory, "", + "Directory where modules with custom query procedures are stored", { + if (value.empty()) return true; + if (utils::DirExists(value)) return true; + std::cout << "Expected --" << flagname << " to point to a directory." + << std::endl; + return false; + }); + using ServerT = communication::Server; using communication::ServerContext; @@ -168,6 +181,16 @@ void SingleNodeMain() { query::InterpreterContext interpreter_context{&db}; SessionData session_data{&db, &interpreter_context, &auth, &audit_log}; + // Register modules + if (!FLAGS_query_modules_directory.empty()) { + for (const auto &entry : + std::filesystem::directory_iterator(FLAGS_query_modules_directory)) { + if (entry.is_regular_file() && entry.path().extension() == ".so") + query::procedure::gModuleRegistry.LoadModuleLibrary(entry.path()); + } + } + // Register modules END + interpreter_context.auth = &auth; ServerContext context; @@ -205,6 +228,7 @@ void SingleNodeMain() { CHECK(server.Start()) << "Couldn't start the Bolt server!"; server.AwaitShutdown(); + query::procedure::gModuleRegistry.UnloadAllModules(); } int main(int argc, char **argv) { return WithInit(argc, argv, SingleNodeMain); }