Integrate loading openCypher module procedures

Summary:
All mgp_* symbols are exported from Memgraph executable, no other
symbols should be visible.

The primary C API header, mg_procedure.h, is now part of the
installation. Also, added a shippable query module example.

Directory `query_modules` is meant to contain sources of modules we
write and ship as part of the installation. Currently, there's only an
example module, but there may be potentially more. Some modules could
only be installed as part of the enterprise release.

For Memgraph to load custom procedures, it needs to be started with a
flag pointing to a directory with compiled shared libraries implementing
those procedures.

Reviewers: mferencevic, ipaljak, llugovic, dsantl, buda

Reviewed By: mferencevic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D2538

CMakeLists.txt

@@ -198,6 +198,7 @@ option(EXPERIMENTAL "Build experimental binaries" OFF)
 option(CUSTOMERS "Build customer binaries" OFF)
 option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
 option(TOOLS "Build tools binaries" ON)
+option(QUERY_MODULES "Build query modules containing custom procedures" ON)
 option(MG_COMMUNITY "Build Memgraph Community Edition" OFF)
 option(ASAN "Build with Address Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
 option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
@@ -296,6 +297,10 @@ if(TOOLS)
   add_subdirectory(tools)
 endif()
 
+if(QUERY_MODULES)
+  add_subdirectory(query_modules)
+endif()
+
 # -----------------------------------------------------------------------------
 
 # ---- Setup CPack --------

include/mg_procedure.syms

@@ -0,0 +1,3 @@
+{
+    mgp_*;
+};

query_modules/CMakeLists.txt

@@ -0,0 +1,19 @@
+# Memgraph Query Modules CMake configuration
+# You should use the top level CMake configuration with -DQUERY_MODULES=ON
+# These modules are meant to be shipped with Memgraph installation.
+
+project(memgraph_query_modules VERSION ${memgraph_VERSION})
+
+disallow_in_source_build()
+
+# Everything that is installed here, should be under the "query_modules" component.
+set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules")
+
+add_library(example SHARED example.c)
+target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/include)
+target_compile_options(example PRIVATE -Wall)
+install(PROGRAMS $<TARGET_FILE:example>
+        DESTINATION lib/memgraph/query_modules
+        RENAME example.so)
+# Also install the source of the example, so user can read it.
+install(FILES example.c DESTINATION lib/memgraph/query_modules)

query_modules/example.c

@@ -0,0 +1,64 @@
+// Compile with clang or gcc:
+// clang -Wall -shared -fPIC -I <path-to-memgraph-include> example.c -o example.so
+// <path-to-memgraph-include> for installed Memgraph will usually be something
+// like `/usr/include/memgraph` or `/usr/local/include/memgraph`.
+// To use the compiled module, you need to run Memgraph configured to load
+// modules from the directory where the compiled module can be found.
+#include "mg_procedure.h"
+
+// This example procedure returns 2 fields: `args` and `result`.
+//   * `args` is a copy of arguments passed to the procedure.
+//   * `result` is the result of this procedure, a "Hello World!" string.
+// In case of memory errors, this function will report them and finish executing.
+//
+// The procedure can be invoked in openCypher using the following call:
+//   CALL example(1, 2, 3) YIELD args, result;
+// Naturally, you may pass in different arguments or yield less fields.
+void mgp_main(const struct mgp_list *args, const struct mgp_graph *graph,
+              struct mgp_result *result, struct mgp_memory *memory) {
+  struct mgp_list *args_copy = mgp_list_make_empty(mgp_list_size(args), memory);
+  if (args_copy == NULL) goto error_memory;
+  for (size_t i = 0; i < mgp_list_size(args); ++i) {
+    int success = mgp_list_append(args_copy, mgp_list_at(args, i));
+    if (!success) goto error_free_list;
+  }
+  struct mgp_result_record *record = mgp_result_new_record(result);
+  if (record == NULL) goto error_free_list;
+  // Transfer ownership of args_copy to mgp_value.
+  struct mgp_value *args_value = mgp_value_make_list(args_copy);
+  if (args_value == NULL) goto error_free_list;
+  int args_inserted = mgp_result_record_insert(record, "args", args_value);
+  // Release `args_value` and contained `args_copy`.
+  mgp_value_destroy(args_value);
+  if (!args_inserted) goto error_memory;
+  struct mgp_value *hello_world_value =
+      mgp_value_make_string("Hello World!", memory);
+  if (hello_world_value == NULL) goto error_memory;
+  int result_inserted =
+      mgp_result_record_insert(record, "result", hello_world_value);
+  mgp_value_destroy(hello_world_value);
+  if (!result_inserted) goto error_memory;
+  // We have successfully finished, so return without error reporting.
+  return;
+
+error_free_list:
+  mgp_list_destroy(args_copy);
+error_memory:
+  mgp_result_set_error_msg(result, "Not enough memory!");
+  return;
+}
+
+// This is an optional function if you need to initialize any global state when
+// your module is loaded.
+int mgp_init_module() {
+  // Return 0 to indicate success.
+  return 0;
+}
+
+// This is an optional function if you need to release any resources before the
+// module is unloaded. You will probably need this if you acquired some
+// resources in mgp_init_module.
+int mgp_shutdown_module() {
+  // Return 0 to indicate success.
+  return 0;
+}

src/CMakeLists.txt

@@ -105,6 +105,10 @@ target_link_libraries(mg-single-node ${MG_SINGLE_NODE_LIBS})
 add_dependencies(mg-single-node generate_opencypher_parser)
 add_dependencies(mg-single-node generate_lcp_single_node)
 target_compile_definitions(mg-single-node PUBLIC MG_SINGLE_NODE)
+# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
+# should be dynamically exported, so that `dlopen` can correctly link the
+# symbols in custom procedure module libraries.
+target_link_libraries(mg-single-node "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
 
 # ----------------------------------------------------------------------------
 # END Memgraph Single Node
@@ -161,6 +165,10 @@ add_dependencies(mg-single-node-v2 generate_lcp_common)
 target_compile_definitions(mg-single-node-v2 PUBLIC MG_SINGLE_NODE_V2)
 add_executable(memgraph-v2 memgraph.cpp)
 target_link_libraries(memgraph-v2 mg-single-node-v2 kvstore_lib telemetry_lib)
+# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
+# should be dynamically exported, so that `dlopen` can correctly link the
+# symbols in custom procedure module libraries.
+target_link_libraries(mg-single-node-v2 "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
 
 # ----------------------------------------------------------------------------
 # END Memgraph Single Node v2
@@ -250,6 +258,12 @@ target_link_libraries(mg-single-node-ha ${MG_SINGLE_NODE_HA_LIBS})
 add_dependencies(mg-single-node-ha generate_opencypher_parser)
 add_dependencies(mg-single-node-ha generate_lcp_single_node_ha)
 target_compile_definitions(mg-single-node-ha PUBLIC MG_SINGLE_NODE_HA)
+# TODO: Make these symbols visible once we add support for custom procedure
+# modules in HA.
+# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
+# should be dynamically exported, so that `dlopen` can correctly link the
+# symbols in custom procedure module libraries.
+# target_link_libraries(mg-single-node-ha "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
 
 # ----------------------------------------------------------------------------
 # END Memgraph Single Node High Availability
@@ -306,6 +320,9 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
 # we cannot use the recommended `install(TARGETS ...)`.
 install(PROGRAMS $<TARGET_FILE:memgraph>
         DESTINATION lib/memgraph RENAME memgraph)
+# Install the include file for writing custom procedures.
+install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h
+        DESTINATION include/memgraph)
 # Install the config file (must use absolute path).
 install(FILES ${CMAKE_SOURCE_DIR}/config/community.conf
         DESTINATION /etc/memgraph RENAME memgraph.conf)

src/memgraph.cpp

@@ -18,8 +18,11 @@
 #include "memgraph_init.hpp"
 #include "query/exceptions.hpp"
 #include "telemetry/telemetry.hpp"
+#include "utils/file.hpp"
 #include "utils/flag_validation.hpp"
 
+#include "query/procedure/module.hpp"
+
 // General purpose flags.
 DEFINE_string(interface, "0.0.0.0",
               "Communication interface on which to listen.");
@@ -88,6 +91,16 @@ DEFINE_VALIDATED_int32(
     "Interval (in milliseconds) used for flushing the audit log buffer.",
     FLAG_IN_RANGE(10, INT32_MAX));
 
+DEFINE_VALIDATED_string(
+    query_modules_directory, "",
+    "Directory where modules with custom query procedures are stored", {
+      if (value.empty()) return true;
+      if (utils::DirExists(value)) return true;
+      std::cout << "Expected --" << flagname << " to point to a directory."
+                << std::endl;
+      return false;
+    });
+
 using ServerT = communication::Server<BoltSession, SessionData>;
 using communication::ServerContext;
 
@@ -168,6 +181,16 @@ void SingleNodeMain() {
   query::InterpreterContext interpreter_context{&db};
   SessionData session_data{&db, &interpreter_context, &auth, &audit_log};
 
+  // Register modules
+  if (!FLAGS_query_modules_directory.empty()) {
+    for (const auto &entry :
+         std::filesystem::directory_iterator(FLAGS_query_modules_directory)) {
+      if (entry.is_regular_file() && entry.path().extension() == ".so")
+        query::procedure::gModuleRegistry.LoadModuleLibrary(entry.path());
+    }
+  }
+  // Register modules END
+
   interpreter_context.auth = &auth;
 
   ServerContext context;
@@ -205,6 +228,7 @@ void SingleNodeMain() {
 
   CHECK(server.Start()) << "Couldn't start the Bolt server!";
   server.AwaitShutdown();
+  query::procedure::gModuleRegistry.UnloadAllModules();
 }
 
 int main(int argc, char **argv) { return WithInit(argc, argv, SingleNodeMain); }