Install and setup loading Python Query Modules

Reviewers: mferencevic

Reviewed By: mferencevic

Subscribers: pullbot, buda

Differential Revision: https://phabricator.memgraph.io/D2715

query_modules/CMakeLists.txt

@@ -27,6 +27,9 @@ install(PROGRAMS $<TARGET_FILE:example>
 # Also install the source of the example, so user can read it.
 install(FILES example.c DESTINATION lib/memgraph/query_modules)
 
+# Install the Python example
+install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
+
 if (MG_ENTERPRISE)
   add_subdirectory(louvain)
   add_subdirectory(connectivity)

release/CMakeLists.txt

@@ -59,7 +59,8 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
  speed, simplicity and scale required to build the next generation of
  applications driver by real-time connected data.")
 # Add `openssl` package to dependencies list. Used to generate SSL certificates.
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0)")
+# We also depend on `python3` because we embed it in Memgraph.
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
 
 # RPM specific
 set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
@@ -81,7 +82,8 @@ set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
 It aims to deliver developers the speed, simplicity and scale required to build
 the next generation of applications driver by real-time connected data.")
 # Add `openssl` package to dependencies list. Used to generate SSL certificates.
-set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0")
+# We also depend on `python3` because we embed it in Memgraph.
+set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0")
 
 # All variables must be set before including.
 include(CPack)

src/CMakeLists.txt

@@ -62,6 +62,11 @@ add_custom_command(TARGET memgraph POST_BUILD
                    COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
                    BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
                    COMMENT "Creating symlink to memgraph executable")
+# Emulate the installed python_support, by creating a symlink
+add_custom_command(TARGET memgraph POST_BUILD
+                   COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
+                   BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
+                   COMMENT "Creating symlink for python_support")
 
 # Strip the executable in release build.
 if (lower_build_type STREQUAL "release")
@@ -92,6 +97,9 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
 # we cannot use the recommended `install(TARGETS ...)`.
 install(PROGRAMS $<TARGET_FILE:memgraph>
         DESTINATION lib/memgraph RENAME memgraph)
+# Install Python source for supporting our embedded Python.
+install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.py
+        DESTINATION lib/memgraph/python_support)
 # Install the include file for writing custom procedures.
 install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h
         DESTINATION include/memgraph)

src/memgraph.cpp

@@ -840,6 +840,26 @@ int main(int argc, char **argv) {
   PyEval_InitThreads();
   Py_BEGIN_ALLOW_THREADS;
 
+  // Add our Python modules to sys.path
+  try {
+    auto exe_path = utils::GetExecutablePath();
+    auto py_support_dir = exe_path.parent_path() / "python_support";
+    if (std::filesystem::is_directory(py_support_dir)) {
+      auto gil = py::EnsureGIL();
+      auto maybe_exc = py::AppendToSysPath(py_support_dir.c_str());
+      if (maybe_exc) {
+        LOG(ERROR) << "Unable to load support for embedded Python: "
+                   << *maybe_exc;
+      }
+    } else {
+      LOG(ERROR)
+          << "Unable to load support for embedded Python: missing directory "
+          << py_support_dir;
+    }
+  } catch (const std::filesystem::filesystem_error &e) {
+    LOG(ERROR) << "Unable to load support for embedded Python: " << e.what();
+  }
+
   // Initialize the communication library.
   communication::Init();