Move essential query modules from MAGE to Memgraph (#1384)

* schema.cpp
* mgps.py
* convert.py

query_modules/CMakeLists.txt

@@ -40,9 +40,26 @@ install(PROGRAMS $<TARGET_FILE:example_cpp>
 # Also install the source of the example, so user can read it.
 install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
 
+add_library(schema SHARED schema.cpp)
+target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include)
+target_compile_options(schema PRIVATE -Wall)
+# Strip C++ example in release build.
+if (lower_build_type STREQUAL "release")
+  add_custom_command(TARGET schema POST_BUILD
+                     COMMAND strip -s $<TARGET_FILE:schema>
+                     COMMENT "Stripping symbols and sections from the C++ schema module")
+endif()
+install(PROGRAMS $<TARGET_FILE:schema>
+        DESTINATION lib/memgraph/query_modules
+        RENAME schema.so)
+# Also install the source of the example, so user can read it.
+install(FILES schema.cpp DESTINATION lib/memgraph/query_modules/src)
+
 # Install the Python example and modules
 install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
 install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
 install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
 install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)
 install(FILES wcc.py DESTINATION lib/memgraph/query_modules)
+install(FILES mgps.py DESTINATION lib/memgraph/query_modules)
+install(FILES convert.py DESTINATION lib/memgraph/query_modules)

query_modules/convert.py

@@ -0,0 +1,10 @@
+from json import loads
+
+import mgp
+
+
+@mgp.function
+def str2object(string: str) -> mgp.Any:
+    if string:
+        return loads(string)
+    return None

query_modules/mgps.py

@@ -0,0 +1,8 @@
+import mgp
+
+
+@mgp.read_proc
+def components(
+    context: mgp.ProcCtx,
+) -> mgp.Record(versions=list, edition=str, name=str):
+    return mgp.Record(versions=["5.9.0"], edition="community", name="Memgraph")

query_modules/schema.cpp

@@ -0,0 +1,187 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <mgp.hpp>
+
+namespace Schema {
+
+/*NodeTypeProperties and RelTypeProperties constants*/
+constexpr std::string_view kReturnNodeType = "nodeType";
+constexpr std::string_view kProcedureNodeType = "node_type_properties";
+constexpr std::string_view kProcedureRelType = "rel_type_properties";
+constexpr std::string_view kReturnLabels = "nodeLabels";
+constexpr std::string_view kReturnRelType = "relType";
+constexpr std::string_view kReturnPropertyName = "propertyName";
+constexpr std::string_view kReturnPropertyType = "propertyTypes";
+constexpr std::string_view kReturnMandatory = "mandatory";
+
+std::string TypeOf(const mgp::Type &type);
+
+template <typename T>
+void ProcessPropertiesNode(mgp::Record &record, const std::string &type, const mgp::List &labels,
+                           const std::string &propertyName, const T &propertyType, const bool &mandatory);
+
+template <typename T>
+void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, const std::string &propertyName,
+                          const T &propertyType, const bool &mandatory);
+
+void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
+void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
+}  // namespace Schema
+
+/*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this
+module, so I implemented a small function here*/
+std::string Schema::TypeOf(const mgp::Type &type) {
+  switch (type) {
+    case mgp::Type::Null:
+      return "Null";
+    case mgp::Type::Bool:
+      return "Bool";
+    case mgp::Type::Int:
+      return "Int";
+    case mgp::Type::Double:
+      return "Double";
+    case mgp::Type::String:
+      return "String";
+    case mgp::Type::List:
+      return "List[Any]";
+    case mgp::Type::Map:
+      return "Map[Any]";
+    case mgp::Type::Node:
+      return "Vertex";
+    case mgp::Type::Relationship:
+      return "Edge";
+    case mgp::Type::Path:
+      return "Path";
+    case mgp::Type::Date:
+      return "Date";
+    case mgp::Type::LocalTime:
+      return "LocalTime";
+    case mgp::Type::LocalDateTime:
+      return "LocalDateTime";
+    case mgp::Type::Duration:
+      return "Duration";
+    default:
+      throw mgp::ValueException("Unsupported type");
+  }
+}
+template <typename T>
+void Schema::ProcessPropertiesNode(mgp::Record &record, const std::string &type, const mgp::List &labels,
+                                   const std::string &propertyName, const T &propertyType, const bool &mandatory) {
+  record.Insert(std::string(kReturnNodeType).c_str(), type);
+  record.Insert(std::string(kReturnLabels).c_str(), labels);
+  record.Insert(std::string(kReturnPropertyName).c_str(), propertyName);
+  record.Insert(std::string(kReturnPropertyType).c_str(), propertyType);
+  record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
+}
+
+template <typename T>
+void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, const std::string &propertyName,
+                                  const T &propertyType, const bool &mandatory) {
+  record.Insert(std::string(kReturnRelType).c_str(), type);
+  record.Insert(std::string(kReturnPropertyName).c_str(), propertyName);
+  record.Insert(std::string(kReturnPropertyType).c_str(), propertyType);
+  record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
+}
+
+void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
+  mgp::MemoryDispatcherGuard guard{memory};
+  ;
+  const auto record_factory = mgp::RecordFactory(result);
+  try {
+    const mgp::Graph graph = mgp::Graph(memgraph_graph);
+    for (auto node : graph.Nodes()) {
+      std::string type = "";
+      mgp::List labels = mgp::List();
+      for (auto label : node.Labels()) {
+        labels.AppendExtend(mgp::Value(label));
+        type += ":`" + std::string(label) + "`";
+      }
+
+      if (node.Properties().size() == 0) {
+        auto record = record_factory.NewRecord();
+        ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
+        continue;
+      }
+
+      for (auto &[key, prop] : node.Properties()) {
+        auto property_type = mgp::List();
+        auto record = record_factory.NewRecord();
+        property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
+        ProcessPropertiesNode<mgp::List>(record, type, labels, key, property_type, true);
+      }
+    }
+
+  } catch (const std::exception &e) {
+    record_factory.SetErrorMessage(e.what());
+    return;
+  }
+}
+
+void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
+  mgp::MemoryDispatcherGuard guard{memory};
+  ;
+  const auto record_factory = mgp::RecordFactory(result);
+  try {
+    const mgp::Graph graph = mgp::Graph(memgraph_graph);
+
+    for (auto rel : graph.Relationships()) {
+      std::string type = ":`" + std::string(rel.Type()) + "`";
+      if (rel.Properties().size() == 0) {
+        auto record = record_factory.NewRecord();
+        ProcessPropertiesRel<std::string>(record, type, "", "", false);
+        continue;
+      }
+
+      for (auto &[key, prop] : rel.Properties()) {
+        auto property_type = mgp::List();
+        auto record = record_factory.NewRecord();
+        property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
+        ProcessPropertiesRel<mgp::List>(record, type, key, property_type, true);
+      }
+    }
+
+  } catch (const std::exception &e) {
+    record_factory.SetErrorMessage(e.what());
+    return;
+  }
+}
+
+extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
+  try {
+    mgp::MemoryDispatcherGuard guard{memory};
+    ;
+
+    AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read,
+                 {},
+                 {mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String),
+                  mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}),
+                  mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
+                  mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
+                  mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
+                 module, memory);
+
+    AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read,
+                 {},
+                 {mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String),
+                  mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
+                  mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
+                  mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
+                 module, memory);
+
+  } catch (const std::exception &e) {
+    return 1;
+  }
+
+  return 0;
+}
+
+extern "C" int mgp_shutdown_module() { return 0; }

tests/e2e/CMakeLists.txt

@@ -66,8 +66,10 @@ add_subdirectory(concurrent_query_modules)
 add_subdirectory(show_index_info)
 add_subdirectory(set_properties)
 add_subdirectory(transaction_rollback)
+add_subdirectory(query_modules)
 add_subdirectory(constraints)
 
 copy_e2e_python_files(pytest_runner pytest_runner.sh "")
+copy_e2e_python_files(x x.sh "")
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/memgraph-selfsigned.crt DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/memgraph-selfsigned.key DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

tests/e2e/query_modules/CMakeLists.txt

@@ -0,0 +1,9 @@
+function(copy_query_modules_e2e_python_files FILE_NAME)
+    copy_e2e_python_files(query_modules ${FILE_NAME})
+endfunction()
+
+copy_query_modules_e2e_python_files(common.py)
+copy_query_modules_e2e_python_files(conftest.py)
+copy_query_modules_e2e_python_files(convert_test.py)
+copy_query_modules_e2e_python_files(mgps_test.py)
+copy_query_modules_e2e_python_files(schema_test.py)

tests/e2e/query_modules/common.py

@@ -0,0 +1,34 @@
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import typing
+
+import mgclient
+
+
+def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]:
+    cursor.execute(query, params)
+    return cursor.fetchall()
+
+
+def connect(**kwargs) -> mgclient.Connection:
+    connection = mgclient.connect(host="localhost", port=7687, **kwargs)
+    connection.autocommit = True
+    return connection
+
+
+def has_n_result_row(cursor: mgclient.Cursor, query: str, n: int):
+    results = execute_and_fetch_all(cursor, query)
+    return len(results) == n
+
+
+def has_one_result_row(cursor: mgclient.Cursor, query: str):
+    return has_n_result_row(cursor, query, 1)

tests/e2e/query_modules/conftest.py

@@ -0,0 +1,21 @@
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import pytest
+from common import connect, execute_and_fetch_all
+
+
+@pytest.fixture(autouse=True)
+def connection():
+    connection = connect()
+    yield connection
+    cursor = connection.cursor()
+    execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n")

tests/e2e/query_modules/convert_test.py

@@ -0,0 +1,31 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import connect, execute_and_fetch_all
+
+
+def test_convert_list1():
+    cursor = connect().cursor()
+    result = execute_and_fetch_all(cursor, f"RETURN convert.str2object('[2, 4, 8, [2]]') AS result;")[0][0]
+    assert (result) == [2, 4, 8, [2]]
+
+
+def test_convert_list_wrong():
+    cursor = connect().cursor()
+    result = execute_and_fetch_all(cursor, f"RETURN convert.str2object('[2, 4, 8, [2]]') AS result;")[0][0]
+    assert (result) != [3, 4, 8, [2]]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/query_modules/mgps_test.py

@@ -0,0 +1,29 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import connect, execute_and_fetch_all
+
+
+def test_mgps1():
+    cursor = connect().cursor()
+    result = list(
+        execute_and_fetch_all(
+            cursor, f"CALL mgps.components() YIELD edition, name, versions RETURN edition, name, versions;"
+        )[0]
+    )
+    assert (result) == ["community", "Memgraph", ["5.9.0"]]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/query_modules/schema_test.py

@@ -0,0 +1,73 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import connect, execute_and_fetch_all
+
+
+def test_node_type_properties1():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        "CREATE (d:Dog {name: 'Rex', owner: 'Carl'})-[l:LOVES]->(a:Activity {name: 'Running', location: 'Zadar'})",
+    )
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+        )[0]
+    )
+    assert (result) == [":`Activity`", ["Activity"], "location", ["String"], True]
+
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+        )[1]
+    )
+    assert (result) == [":`Activity`", ["Activity"], "name", ["String"], True]
+
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+        )[2]
+    )
+    assert (result) == [":`Dog`", ["Dog"], "name", ["String"], True]
+
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+        )[3]
+    )
+    assert (result) == [":`Dog`", ["Dog"], "owner", ["String"], True]
+
+
+def test_rel_type_properties1():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        "CREATE (d:Dog {name: 'Rex', owner: 'Carl'})-[l:LOVES]->(a:Activity {name: 'Running', location: 'Zadar'})",
+    )
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;",
+        )[0]
+    )
+    assert (result) == [":`LOVES`", "", "", False]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/query_modules/workloads.yaml

@@ -0,0 +1,35 @@
+bolt_port: &bolt_port "7687"
+args: &args
+  - "--bolt_port"
+  - *bolt_port
+  - "--log-level=TRACE"
+
+in_memory_cluster: &in_memory_cluster
+  cluster:
+    main:
+      args: *args
+      log_file: "query_modules-e2e.log"
+      setup_queries: []
+      validation_queries: []
+
+workloads:
+  - name: "Convert query module test"
+    pre_set_workload: "tests/e2e/x.sh"
+    binary: "tests/e2e/pytest_runner.sh"
+    proc: "query_modules/"
+    args: ["query_modules/convert_test.py"]
+    <<: *in_memory_cluster
+
+  - name: "Mgps query module test"
+    pre_set_workload: "tests/e2e/x.sh"
+    binary: "tests/e2e/pytest_runner.sh"
+    proc: "query_modules/"
+    args: ["query_modules/mgps_test.py"]
+    <<: *in_memory_cluster
+
+  - name: "Convert query module test"
+    pre_set_workload: "tests/e2e/x.sh"
+    binary: "tests/e2e/pytest_runner.sh"
+    proc: "query_modules/"
+    args: ["query_modules/schema_test.py"]
+    <<: *in_memory_cluster

tests/e2e/runner.py

@@ -56,6 +56,10 @@ def run(args):
         def cleanup():
             interactive_mg_runner.stop_all()
 
+        if "pre_set_workload" in workload:
+            binary = os.path.join(BUILD_DIR, workload["pre_set_workload"])
+            subprocess.run([binary], check=True, stderr=subprocess.STDOUT)
+
         if "cluster" in workload:
             procdir = ""
             if "proc" in workload:

tests/e2e/x.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp ../../query_modules/convert.py ../../build/query_modules
+cp ../../query_modules/mgps.py ../../build/query_modules