From 23171e76b6620429ff971a5abd8cf61bf7538542 Mon Sep 17 00:00:00 2001 From: Jure Bajic Date: Tue, 11 Oct 2022 16:31:46 +0200 Subject: [PATCH] Integrate bolt server (#572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use query-v2 in the main executable * Set up machine manager in memgraph * Add `ShardRequestManager` to `Interpreter` * Make vertex creation work * Make scan all work * Add edge type map in shard request manager * Send schema over request * Empty out DbAccessor * Store shard mapping at creation * Remove failing CI steps Cooltura is the best place in Zagreb! Co-authored-by: János Benjamin Antal --- .github/workflows/diff.yaml | 231 +- config/flags.yaml | 10 +- src/CMakeLists.txt | 24 +- src/communication/v2/listener.hpp | 7 +- src/communication/v2/server.hpp | 2 +- src/communication/v2/session.hpp | 11 +- src/coordinator/shard_map.cpp | 3 +- src/coordinator/shard_map.hpp | 90 +- src/glue/v2/communication.cpp | 154 +- src/glue/v2/communication.hpp | 27 +- src/io/future.hpp | 3 +- src/machine_manager/machine_manager.hpp | 5 +- src/memgraph.cpp | 801 +---- src/query/v2/CMakeLists.txt | 10 - src/query/v2/accessors.cpp | 3 +- src/query/v2/accessors.hpp | 3 +- src/query/v2/common.hpp | 65 - src/query/v2/context.hpp | 24 +- src/query/v2/cypher_query_interpreter.cpp | 9 +- src/query/v2/cypher_query_interpreter.hpp | 4 +- src/query/v2/db_accessor.hpp | 188 -- src/query/v2/dump.cpp | 483 --- src/query/v2/dump.hpp | 63 - src/query/v2/frontend/ast/ast.lcp | 1 + .../interpret/awesome_memgraph_functions.cpp | 79 +- src/query/v2/interpreter.cpp | 130 +- src/query/v2/interpreter.hpp | 13 +- src/query/v2/path.hpp | 4 + src/query/v2/plan/operator.cpp | 676 +--- src/query/v2/plan/pretty_print.cpp | 94 +- src/query/v2/plan/pretty_print.hpp | 34 +- src/query/v2/plan/vertex_count_cache.hpp | 118 +- src/query/v2/procedure/cypher_type_ptr.hpp | 20 - src/query/v2/procedure/cypher_types.hpp | 293 -- .../v2/procedure/mg_procedure_helpers.cpp | 36 - .../v2/procedure/mg_procedure_helpers.hpp | 69 - src/query/v2/procedure/mg_procedure_impl.cpp | 2812 ----------------- src/query/v2/procedure/mg_procedure_impl.hpp | 927 ------ src/query/v2/procedure/module.cpp | 1258 -------- src/query/v2/procedure/module.hpp | 246 -- src/query/v2/procedure/py_module.cpp | 2650 ---------------- src/query/v2/procedure/py_module.hpp | 82 - src/query/v2/requests.hpp | 1 + src/query/v2/shard_request_manager.hpp | 68 +- src/query/v2/stream/common.cpp | 45 - src/query/v2/stream/common.hpp | 87 - src/query/v2/stream/sources.cpp | 137 - src/query/v2/stream/sources.hpp | 95 - src/query/v2/stream/streams.cpp | 773 ----- src/query/v2/stream/streams.hpp | 206 -- src/query/v2/trigger.cpp | 6 +- src/query/v2/trigger_context.hpp | 4 +- src/storage/v3/bindings/db_accessor.hpp | 3 +- src/storage/v3/shard.cpp | 22 +- src/storage/v3/shard.hpp | 5 +- src/storage/v3/shard_manager.hpp | 16 +- src/storage/v3/shard_rsm.cpp | 45 +- tests/unit/machine_manager.cpp | 2 +- tests/unit/query_v2_dummy_test.cpp | 1 - 59 files changed, 711 insertions(+), 12567 deletions(-) delete mode 100644 src/query/v2/dump.cpp delete mode 100644 src/query/v2/dump.hpp delete mode 100644 src/query/v2/procedure/cypher_type_ptr.hpp delete mode 100644 src/query/v2/procedure/cypher_types.hpp delete mode 100644 src/query/v2/procedure/mg_procedure_helpers.cpp delete mode 100644 src/query/v2/procedure/mg_procedure_helpers.hpp delete mode 100644 src/query/v2/procedure/mg_procedure_impl.cpp delete mode 100644 src/query/v2/procedure/mg_procedure_impl.hpp delete mode 100644 src/query/v2/procedure/module.cpp delete mode 100644 src/query/v2/procedure/module.hpp delete mode 100644 src/query/v2/procedure/py_module.cpp delete mode 100644 src/query/v2/procedure/py_module.hpp delete mode 100644 src/query/v2/stream/common.cpp delete mode 100644 src/query/v2/stream/common.hpp delete mode 100644 src/query/v2/stream/sources.cpp delete mode 100644 src/query/v2/stream/sources.hpp delete mode 100644 src/query/v2/stream/streams.cpp delete mode 100644 src/query/v2/stream/streams.hpp diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index e13fed516..00aca7f5c 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -164,52 +164,14 @@ jobs: cmake .. make -j$THREADS - - name: Run leftover CTest tests + - name: Run simulation tests run: | # Activate toolchain. source /opt/toolchain-v4/activate - # Run leftover CTest tests (all except unit and benchmark tests). + # Run simulation tests. cd build - ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure - - - name: Run drivers tests - run: | - ./tests/drivers/run.sh - - - name: Run integration tests - run: | - cd tests/integration - for name in *; do - if [ ! -d $name ]; then continue; fi - pushd $name >/dev/null - echo "Running: $name" - if [ -x prepare.sh ]; then - ./prepare.sh - fi - if [ -x runner.py ]; then - ./runner.py - elif [ -x runner.sh ]; then - ./runner.sh - fi - echo - popd >/dev/null - done - - - name: Run cppcheck and clang-format - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Run cppcheck and clang-format. - cd tools/github - ./cppcheck_and_clang_format diff - - - name: Save cppcheck and clang-format errors - uses: actions/upload-artifact@v2 - with: - name: "Code coverage" - path: tools/github/cppcheck_and_clang_format.txt + ctest -R memgraph__simulation --output-on-failure -j$THREADS release_build: name: "Release build" @@ -240,19 +202,6 @@ jobs: cmake -DCMAKE_BUILD_TYPE=release .. make -j$THREADS - - name: Run GQL Behave tests - run: | - cd tests/gql_behave - ./continuous_integration - - - name: Save quality assurance status - uses: actions/upload-artifact@v2 - with: - name: "GQL Behave Status" - path: | - tests/gql_behave/gql_behave_status.csv - tests/gql_behave/gql_behave_status.html - - name: Run unit tests run: | # Activate toolchain. @@ -267,178 +216,6 @@ jobs: # Activate toolchain. source /opt/toolchain-v4/activate - # Run unit tests. + # Run simulation tests. cd build ctest -R memgraph__simulation --output-on-failure -j$THREADS - - - name: Run e2e tests - run: | - # TODO(gitbuda): Setup mgclient and pymgclient properly. - cd tests - ./setup.sh - source ve3/bin/activate - cd e2e - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory . - - - name: Run stress test (plain) - run: | - cd tests/stress - ./continuous_integration - - - name: Run stress test (SSL) - run: | - cd tests/stress - ./continuous_integration --use-ssl - - - name: Run durability test - run: | - cd tests/stress - source ve3/bin/activate - python3 durability --num-steps 5 - - - name: Create enterprise DEB package - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - cd build - - # create mgconsole - # we use the -B to force the build - make -j$THREADS -B mgconsole - - # Create enterprise DEB package. - mkdir output && cd output - cpack -G DEB --config ../CPackConfig.cmake - - - name: Save enterprise DEB package - uses: actions/upload-artifact@v2 - with: - name: "Enterprise DEB package" - path: build/output/memgraph*.deb - - - name: Save test data - uses: actions/upload-artifact@v2 - if: always() - with: - name: "Test data" - path: | - # multiple paths could be defined - build/logs - - release_jepsen_test: - name: "Release Jepsen Test" - runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl] - #continue-on-error: true - env: - THREADS: 24 - MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} - MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} - - steps: - - name: Set up repository - uses: actions/checkout@v2 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build only memgraph release binarie. - cd build - cmake -DCMAKE_BUILD_TYPE=release .. - make -j$THREADS memgraph - - - name: Run Jepsen tests - run: | - cd tests/jepsen - ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs - - - name: Save Jepsen report - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: "Jepsen Report" - path: tests/jepsen/Jepsen.tar.gz - - release_benchmarks: - name: "Release benchmarks" - runs-on: [self-hosted, Linux, X64, Diff, Gen7] - env: - THREADS: 24 - MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} - MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} - - steps: - - name: Set up repository - uses: actions/checkout@v2 - with: - # Number of commits to fetch. `0` indicates all history for all - # branches and tags. (default: 1) - fetch-depth: 0 - - - name: Build release binaries - run: | - # Activate toolchain. - source /opt/toolchain-v4/activate - - # Initialize dependencies. - ./init - - # Build only memgraph release binaries. - cd build - cmake -DCMAKE_BUILD_TYPE=release .. - make -j$THREADS - - - name: Run macro benchmarks - run: | - cd tests/macro_benchmark - ./harness QuerySuite MemgraphRunner \ - --groups aggregation 1000_create unwind_create dense_expand match \ - --no-strict - - - name: Get branch name (merge) - if: github.event_name != 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV - - - name: Get branch name (pull request) - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV - - - name: Upload macro benchmark results - run: | - cd tools/bench-graph-client - virtualenv -p python3 ve3 - source ve3/bin/activate - pip install -r requirements.txt - ./main.py --benchmark-name "macro_benchmark" \ - --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \ - --github-run-id "${{ github.run_id }}" \ - --github-run-number "${{ github.run_number }}" \ - --head-branch-name "${{ env.BRANCH_NAME }}" - - - name: Run mgbench - run: | - cd tests/mgbench - ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/* - - - name: Upload mgbench results - run: | - cd tools/bench-graph-client - virtualenv -p python3 ve3 - source ve3/bin/activate - pip install -r requirements.txt - ./main.py --benchmark-name "mgbench" \ - --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \ - --github-run-id "${{ github.run_id }}" \ - --github-run-number "${{ github.run_number }}" \ - --head-branch-name "${{ env.BRANCH_NAME }}" diff --git a/config/flags.yaml b/config/flags.yaml index 6dd50e50c..65bc1ea28 100644 --- a/config/flags.yaml +++ b/config/flags.yaml @@ -83,13 +83,9 @@ modifications: value: "true" override: true - - name: "query_modules_directory" - value: "/usr/lib/memgraph/query_modules" - override: true - - - name: "auth_module_executable" - value: "/usr/lib/memgraph/auth_module/example.py" - override: false + # - name: "query_modules_directory" + # value: "/usr/lib/memgraph/query_modules" + # override: true - name: "memory_limit" value: "0" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8eab86bc9..8b84bb26a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -37,13 +37,13 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) # Memgraph Single Node v2 Executable # ---------------------------------------------------------------------------- set(mg_single_node_v2_sources - glue/communication.cpp - memgraph.cpp - glue/auth.cpp + glue/v2/communication.cpp + memgraph.cpp + glue/v2/auth.cpp ) set(mg_single_node_v2_libs stdc++fs Threads::Threads - telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings) + telemetry_lib mg-query-v2 mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-io mg-coordinator) if (MG_ENTERPRISE) # These are enterprise subsystems set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit) @@ -126,19 +126,3 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph # ---------------------------------------------------------------------------- # Memgraph CSV Import Tool Executable # ---------------------------------------------------------------------------- - -add_executable(mg_import_csv mg_import_csv.cpp) -target_link_libraries(mg_import_csv mg-storage-v2) - -# Strip the executable in release build. -if (lower_build_type STREQUAL "release") - add_custom_command(TARGET mg_import_csv POST_BUILD - COMMAND strip -s mg_import_csv - COMMENT "Stripping symbols and sections from mg_import_csv") -endif() - -install(TARGETS mg_import_csv RUNTIME DESTINATION bin) - -# ---------------------------------------------------------------------------- -# Memgraph CSV Import Tool Executable -# ---------------------------------------------------------------------------- diff --git a/src/communication/v2/listener.hpp b/src/communication/v2/listener.hpp index bd26bea12..4b02ae4cb 100644 --- a/src/communication/v2/listener.hpp +++ b/src/communication/v2/listener.hpp @@ -30,6 +30,7 @@ #include "communication/context.hpp" #include "communication/v2/pool.hpp" #include "communication/v2/session.hpp" +#include "utils/message.hpp" #include "utils/spin_lock.hpp" #include "utils/synchronized.hpp" @@ -58,10 +59,10 @@ class Listener final : public std::enable_shared_from_thisStart(); DoAccept(); diff --git a/src/communication/v2/server.hpp b/src/communication/v2/server.hpp index 0165035b4..363fc5f93 100644 --- a/src/communication/v2/server.hpp +++ b/src/communication/v2/server.hpp @@ -72,7 +72,7 @@ class Server final { * Constructs and binds server to endpoint, operates on session data and * invokes workers_count workers */ - Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context, + Server(ServerEndpoint &endpoint, TSessionData &session_data, ServerContext *server_context, const int inactivity_timeout_sec, const std::string_view service_name, size_t workers_count = std::thread::hardware_concurrency()) : endpoint_{endpoint}, diff --git a/src/communication/v2/session.hpp b/src/communication/v2/session.hpp index 7c8cd30d7..c3d308742 100644 --- a/src/communication/v2/session.hpp +++ b/src/communication/v2/session.hpp @@ -41,6 +41,7 @@ #include #include +#include "communication/buffer.hpp" #include "communication/context.hpp" #include "communication/exceptions.hpp" #include "utils/logging.hpp" @@ -139,7 +140,7 @@ class WebsocketSession : public std::enable_shared_from_this(socket_)) { auto sock = std::get(std::move(socket_)); - WebsocketSession::Create(std::move(sock), data_, endpoint_, service_name_) + WebsocketSession::Create(std::move(sock), *data_, endpoint_, service_name_) ->DoAccept(parser.release()); execution_active_ = false; return; @@ -465,7 +466,7 @@ class Session final : public std::enable_shared_from_this ShardMap::InitializeNewLabel(std::string label_name, std: }; LabelSpace label_space{ - .schema = std::move(schema), + .schema = schema, .shards = shards, .replication_factor = replication_factor, }; + schemas[label_id] = std::move(schema); label_spaces.emplace(label_id, label_space); diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index b416ea330..f99bdbc25 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -11,6 +11,7 @@ #pragma once +#include #include #include #include @@ -27,6 +28,7 @@ #include "storage/v3/property_value.hpp" #include "storage/v3/schemas.hpp" #include "storage/v3/temporal.hpp" +#include "utils/exceptions.hpp" namespace memgraph::coordinator { @@ -34,6 +36,7 @@ constexpr int64_t kNotExistingId{0}; using memgraph::io::Address; using memgraph::storage::v3::Config; +using memgraph::storage::v3::EdgeTypeId; using memgraph::storage::v3::LabelId; using memgraph::storage::v3::PropertyId; using memgraph::storage::v3::PropertyValue; @@ -58,7 +61,9 @@ using Shard = std::vector; using Shards = std::map; using LabelName = std::string; using PropertyName = std::string; +using EdgeTypeName = std::string; using PropertyMap = std::map; +using EdgeTypeIdMap = std::map; struct ShardToInitialize { boost::uuids::uuid uuid; @@ -80,7 +85,9 @@ struct LabelSpace { struct ShardMap { Hlc shard_map_version; uint64_t max_property_id{kNotExistingId}; + uint64_t max_edge_type_id{kNotExistingId}; std::map properties; + std::map edge_types; uint64_t max_label_id{kNotExistingId}; std::map labels; std::map label_spaces; @@ -127,7 +134,7 @@ struct ShardMap { .label_id = label_id, .min_key = low_key, .max_key = std::nullopt, - .schema = label_space.schema, + .schema = schemas[label_id], .config = Config{}, }); } @@ -140,13 +147,12 @@ struct ShardMap { // TODO(tyler) use deterministic UUID so that coordinators don't diverge here address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()}, - ret.push_back(ShardToInitialize{ - .uuid = address.unique_id, - .label_id = label_id, - .min_key = low_key, - .max_key = std::nullopt, - .config = Config{}, - }); + ret.push_back(ShardToInitialize{.uuid = address.unique_id, + .label_id = label_id, + .min_key = low_key, + .max_key = std::nullopt, + .schema = schemas[label_id], + .config = Config{}}); AddressAndStatus aas = { .address = address, @@ -196,6 +202,49 @@ struct ShardMap { LabelId GetLabelId(const std::string &label) const { return labels.at(label); } + std::string GetLabelName(const LabelId label) const { + if (const auto it = + std::ranges::find_if(labels, [label](const auto &name_id_pair) { return name_id_pair.second == label; }); + it != labels.end()) { + return it->first; + } + throw utils::BasicException("GetLabelName fails on the given label id!"); + } + + std::optional GetPropertyId(const std::string &property_name) const { + if (properties.contains(property_name)) { + return properties.at(property_name); + } + + return std::nullopt; + } + + std::string GetPropertyName(const PropertyId property) const { + if (const auto it = std::ranges::find_if( + properties, [property](const auto &name_id_pair) { return name_id_pair.second == property; }); + it != properties.end()) { + return it->first; + } + throw utils::BasicException("PropertyId not found!"); + } + + std::optional GetEdgeTypeId(const std::string &edge_type) const { + if (edge_types.contains(edge_type)) { + return edge_types.at(edge_type); + } + + return std::nullopt; + } + + std::string GetEdgeTypeName(const EdgeTypeId property) const { + if (const auto it = std::ranges::find_if( + edge_types, [property](const auto &name_id_pair) { return name_id_pair.second == property; }); + it != edge_types.end()) { + return it->first; + } + throw utils::BasicException("EdgeTypeId not found!"); + } + Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const { MG_ASSERT(start_key <= end_key); MG_ASSERT(labels.contains(label_name)); @@ -268,12 +317,29 @@ struct ShardMap { return ret; } - std::optional GetPropertyId(const std::string &property_name) const { - if (properties.contains(property_name)) { - return properties.at(property_name); + EdgeTypeIdMap AllocateEdgeTypeIds(const std::vector &new_edge_types) { + EdgeTypeIdMap ret; + + bool mutated = false; + + for (const auto &edge_type_name : new_edge_types) { + if (edge_types.contains(edge_type_name)) { + auto edge_type_id = edge_types.at(edge_type_name); + ret.emplace(edge_type_name, edge_type_id); + } else { + mutated = true; + + const EdgeTypeId edge_type_id = EdgeTypeId::FromUint(++max_edge_type_id); + ret.emplace(edge_type_name, edge_type_id); + edge_types.emplace(edge_type_name, edge_type_id); + } } - return std::nullopt; + if (mutated) { + IncrementShardMapVersion(); + } + + return ret; } }; diff --git a/src/glue/v2/communication.cpp b/src/glue/v2/communication.cpp index d05b521fe..217f39545 100644 --- a/src/glue/v2/communication.cpp +++ b/src/glue/v2/communication.cpp @@ -15,9 +15,16 @@ #include #include +#include "coordinator/shard_map.hpp" +#include "query/v2/accessors.hpp" +#include "query/v2/requests.hpp" #include "storage/v3/edge_accessor.hpp" +#include "storage/v3/id_types.hpp" +#include "storage/v3/result.hpp" #include "storage/v3/shard.hpp" #include "storage/v3/vertex_accessor.hpp" +#include "storage/v3/view.hpp" +#include "utils/exceptions.hpp" #include "utils/temporal.hpp" using memgraph::communication::bolt::Value; @@ -63,17 +70,51 @@ query::v2::TypedValue ToTypedValue(const Value &value) { } } -storage::v3::Result ToBoltVertex(const query::v2::VertexAccessor &vertex, - const storage::v3::Shard &db, storage::v3::View view) { - return ToBoltVertex(vertex.impl_, db, view); +storage::v3::Result ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex, + const coordinator::ShardMap &shard_map, + storage::v3::View /*view*/) { + auto id = communication::bolt::Id::FromUint(0); + + auto labels = vertex.Labels(); + std::vector new_labels; + new_labels.reserve(labels.size()); + for (const auto &label : labels) { + new_labels.push_back(shard_map.GetLabelName(label.id)); + } + + auto properties = vertex.Properties(); + std::map new_properties; + for (const auto &[prop, property_value] : properties) { + new_properties[shard_map.GetPropertyName(prop)] = ToBoltValue(property_value); + } + return communication::bolt::Vertex{id, new_labels, new_properties}; } -storage::v3::Result ToBoltEdge(const query::v2::EdgeAccessor &edge, - const storage::v3::Shard &db, storage::v3::View view) { - return ToBoltEdge(edge.impl_, db, view); +storage::v3::Result ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge, + const coordinator::ShardMap &shard_map, + storage::v3::View /*view*/) { + // TODO(jbajic) Fix bolt communication + auto id = communication::bolt::Id::FromUint(0); + auto from = communication::bolt::Id::FromUint(0); + auto to = communication::bolt::Id::FromUint(0); + const auto &type = shard_map.GetEdgeTypeName(edge.EdgeType()); + + auto properties = edge.Properties(); + std::map new_properties; + for (const auto &[prop, property_value] : properties) { + new_properties[shard_map.GetPropertyName(prop)] = ToBoltValue(property_value); + } + return communication::bolt::Edge{id, from, to, type, new_properties}; } -storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db, +storage::v3::Result ToBoltPath(const query::v2::accessors::Path & /*edge*/, + const coordinator::ShardMap & /*shard_map*/, + storage::v3::View /*view*/) { + // TODO(jbajic) Fix bolt communication + return {storage::v3::Error::DELETED_OBJECT}; +} + +storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, const coordinator::ShardMap &shard_map, storage::v3::View view) { switch (value.type()) { case query::v2::TypedValue::Type::Null: @@ -90,7 +131,7 @@ storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, const std::vector values; values.reserve(value.ValueList().size()); for (const auto &v : value.ValueList()) { - auto maybe_value = ToBoltValue(v, db, view); + auto maybe_value = ToBoltValue(v, shard_map, view); if (maybe_value.HasError()) return maybe_value.GetError(); values.emplace_back(std::move(*maybe_value)); } @@ -99,24 +140,24 @@ storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, const case query::v2::TypedValue::Type::Map: { std::map map; for (const auto &kv : value.ValueMap()) { - auto maybe_value = ToBoltValue(kv.second, db, view); + auto maybe_value = ToBoltValue(kv.second, shard_map, view); if (maybe_value.HasError()) return maybe_value.GetError(); map.emplace(kv.first, std::move(*maybe_value)); } return Value(std::move(map)); } case query::v2::TypedValue::Type::Vertex: { - auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view); + auto maybe_vertex = ToBoltVertex(value.ValueVertex(), shard_map, view); if (maybe_vertex.HasError()) return maybe_vertex.GetError(); return Value(std::move(*maybe_vertex)); } case query::v2::TypedValue::Type::Edge: { - auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view); + auto maybe_edge = ToBoltEdge(value.ValueEdge(), shard_map, view); if (maybe_edge.HasError()) return maybe_edge.GetError(); return Value(std::move(*maybe_edge)); } case query::v2::TypedValue::Type::Path: { - auto maybe_path = ToBoltPath(value.ValuePath(), db, view); + auto maybe_path = ToBoltPath(value.ValuePath(), shard_map, view); if (maybe_path.HasError()) return maybe_path.GetError(); return Value(std::move(*maybe_path)); } @@ -131,59 +172,48 @@ storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, const } } -storage::v3::Result ToBoltVertex(const storage::v3::VertexAccessor &vertex, - const storage::v3::Shard &db, storage::v3::View view) { - // TODO(jbajic) Fix bolt communication - auto id = communication::bolt::Id::FromUint(0); - auto maybe_labels = vertex.Labels(view); - if (maybe_labels.HasError()) return maybe_labels.GetError(); - std::vector labels; - labels.reserve(maybe_labels->size()); - for (const auto &label : *maybe_labels) { - labels.push_back(db.LabelToName(label)); +Value ToBoltValue(msgs::Value value) { + switch (value.type) { + case msgs::Value::Type::Null: + return {}; + case msgs::Value::Type::Bool: + return {value.bool_v}; + case msgs::Value::Type::Int64: + return {value.int_v}; + case msgs::Value::Type::Double: + return {value.double_v}; + case msgs::Value::Type::String: + return {std::string(value.string_v)}; + case msgs::Value::Type::List: { + std::vector values; + values.reserve(value.list_v.size()); + for (const auto &v : value.list_v) { + auto maybe_value = ToBoltValue(v); + values.emplace_back(std::move(maybe_value)); + } + return Value{std::move(values)}; + } + case msgs::Value::Type::Map: { + std::map map; + for (const auto &kv : value.map_v) { + auto maybe_value = ToBoltValue(kv.second); + map.emplace(kv.first, std::move(maybe_value)); + } + return Value{std::move(map)}; + } + case msgs::Value::Type::Vertex: + case msgs::Value::Type::Edge: + case msgs::Value::Type::Path: { + throw utils::BasicException("Path, Vertex and Edge not supported!"); + } + // TODO Value to Date types not supported } - auto maybe_properties = vertex.Properties(view); - if (maybe_properties.HasError()) return maybe_properties.GetError(); - std::map properties; - for (const auto &prop : *maybe_properties) { - properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second); - } - return communication::bolt::Vertex{id, labels, properties}; } -storage::v3::Result ToBoltEdge(const storage::v3::EdgeAccessor &edge, - const storage::v3::Shard &db, storage::v3::View view) { - // TODO(jbajic) Fix bolt communication - auto id = communication::bolt::Id::FromUint(0); - auto from = communication::bolt::Id::FromUint(0); - auto to = communication::bolt::Id::FromUint(0); - const auto &type = db.EdgeTypeToName(edge.EdgeType()); - auto maybe_properties = edge.Properties(view); - if (maybe_properties.HasError()) return maybe_properties.GetError(); - std::map properties; - for (const auto &prop : *maybe_properties) { - properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second); - } - return communication::bolt::Edge{id, from, to, type, properties}; -} - -storage::v3::Result ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db, - storage::v3::View view) { - std::vector vertices; - vertices.reserve(path.vertices().size()); - for (const auto &v : path.vertices()) { - auto maybe_vertex = ToBoltVertex(v, db, view); - if (maybe_vertex.HasError()) return maybe_vertex.GetError(); - vertices.emplace_back(std::move(*maybe_vertex)); - } - std::vector edges; - edges.reserve(path.edges().size()); - for (const auto &e : path.edges()) { - auto maybe_edge = ToBoltEdge(e, db, view); - if (maybe_edge.HasError()) return maybe_edge.GetError(); - edges.emplace_back(std::move(*maybe_edge)); - } - return communication::bolt::Path(vertices, edges); +storage::v3::Result ToBoltPath(const query::v2::accessors::Path & /*path*/, + const storage::v3::Shard & /*db*/, + storage::v3::View /*view*/) { + return communication::bolt::Path(); } storage::v3::PropertyValue ToPropertyValue(const Value &value) { diff --git a/src/glue/v2/communication.hpp b/src/glue/v2/communication.hpp index 40e2687ac..67a951c6f 100644 --- a/src/glue/v2/communication.hpp +++ b/src/glue/v2/communication.hpp @@ -13,9 +13,11 @@ #pragma once #include "communication/bolt/v1/value.hpp" +#include "coordinator/shard_map.hpp" #include "query/v2/bindings/typed_value.hpp" #include "storage/v3/property_value.hpp" #include "storage/v3/result.hpp" +#include "storage/v3/shard.hpp" #include "storage/v3/view.hpp" namespace memgraph::storage::v3 { @@ -28,36 +30,40 @@ namespace memgraph::glue::v2 { /// @param storage::v3::VertexAccessor for converting to /// communication::bolt::Vertex. -/// @param storage::v3::Shard for getting label and property names. +/// @param coordinator::ShardMap shard_map getting label and property names. /// @param storage::v3::View for deciding which vertex attributes are visible. /// /// @throw std::bad_alloc storage::v3::Result ToBoltVertex(const storage::v3::VertexAccessor &vertex, - const storage::v3::Shard &db, storage::v3::View view); + const coordinator::ShardMap &shard_map, + storage::v3::View view); /// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge. -/// @param storage::v3::Shard for getting edge type and property names. +/// @param coordinator::ShardMap shard_map getting edge type and property names. /// @param storage::v3::View for deciding which edge attributes are visible. /// /// @throw std::bad_alloc storage::v3::Result ToBoltEdge(const storage::v3::EdgeAccessor &edge, - const storage::v3::Shard &db, storage::v3::View view); + const coordinator::ShardMap &shard_map, + storage::v3::View view); /// @param query::v2::Path for converting to communication::bolt::Path. -/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge. +/// @param coordinator::ShardMap shard_map ToBoltVertex and ToBoltEdge. /// @param storage::v3::View for ToBoltVertex and ToBoltEdge. /// /// @throw std::bad_alloc -storage::v3::Result ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db, +storage::v3::Result ToBoltPath(const query::v2::accessors::Path &path, + const coordinator::ShardMap &shard_map, storage::v3::View view); /// @param query::v2::TypedValue for converting to communication::bolt::Value. -/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge. +/// @param coordinator::ShardMap shard_map ToBoltVertex and ToBoltEdge. /// @param storage::v3::View for ToBoltVertex and ToBoltEdge. /// /// @throw std::bad_alloc storage::v3::Result ToBoltValue(const query::v2::TypedValue &value, - const storage::v3::Shard &db, storage::v3::View view); + const coordinator::ShardMap &shard_map, + storage::v3::View view); query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value); @@ -65,4 +71,9 @@ communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value); storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value); +communication::bolt::Value ToBoltValue(msgs::Value value); + +communication::bolt::Value ToBoltValue(msgs::Value value, const coordinator::ShardMap &shard_map, + storage::v3::View view); + } // namespace memgraph::glue::v2 diff --git a/src/io/future.hpp b/src/io/future.hpp index c697670ab..14fe813d1 100644 --- a/src/io/future.hpp +++ b/src/io/future.hpp @@ -192,7 +192,7 @@ class Future { template class Promise { std::shared_ptr> shared_; - bool filled_or_moved_ = false; + bool filled_or_moved_{false}; public: explicit Promise(std::shared_ptr> shared) : shared_(shared) {} @@ -212,6 +212,7 @@ class Promise { Promise(const Promise &) = delete; Promise &operator=(const Promise &) = delete; + // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Branch) ~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); } // Fill the expected item into the Future. diff --git a/src/machine_manager/machine_manager.hpp b/src/machine_manager/machine_manager.hpp index 5be419189..ea5b0dff9 100644 --- a/src/machine_manager/machine_manager.hpp +++ b/src/machine_manager/machine_manager.hpp @@ -18,6 +18,7 @@ #include #include #include +#include "coordinator/shard_map.hpp" namespace memgraph::machine_manager { @@ -69,11 +70,11 @@ class MachineManager { public: // TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress // which is only true for single-machine config. - MachineManager(io::Io io, MachineConfig config, Coordinator coordinator) + MachineManager(io::Io io, MachineConfig config, Coordinator coordinator, coordinator::ShardMap &shard_map) : io_(io), config_(config), coordinator_{std::move(io.ForkLocal()), {}, std::move(coordinator)}, - shard_manager_(ShardManager{io.ForkLocal(), coordinator_.GetAddress()}) {} + shard_manager_{io.ForkLocal(), coordinator_.GetAddress(), shard_map} {} Address CoordinatorAddress() { return coordinator_.GetAddress(); } diff --git a/src/memgraph.cpp b/src/memgraph.cpp index d4fc3bad1..023135478 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -35,20 +35,28 @@ #include "communication/bolt/v1/constants.hpp" #include "communication/websocket/auth.hpp" #include "communication/websocket/server.hpp" +#include "coordinator/shard_map.hpp" #include "helpers.hpp" +#include "io/address.hpp" +#include "io/local_transport/local_system.hpp" +#include "io/local_transport/local_transport.hpp" +#include "io/simulator/simulator_transport.hpp" +#include "machine_manager/machine_config.hpp" +#include "machine_manager/machine_manager.hpp" #include "py/py.hpp" -#include "query/auth_checker.hpp" -#include "query/discard_value_stream.hpp" -#include "query/exceptions.hpp" -#include "query/frontend/ast/ast.hpp" -#include "query/interpreter.hpp" -#include "query/plan/operator.hpp" -#include "query/procedure/module.hpp" -#include "query/procedure/py_module.hpp" +#include "query/v2/discard_value_stream.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/interpreter.hpp" +#include "query/v2/plan/operator.hpp" #include "requests/requests.hpp" -#include "storage/v2/isolation_level.hpp" -#include "storage/v2/storage.hpp" -#include "storage/v2/view.hpp" +#include "storage/v3/id_types.hpp" +#include "storage/v3/isolation_level.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/schemas.hpp" +#include "storage/v3/shard.hpp" +#include "storage/v3/view.hpp" #include "telemetry/telemetry.hpp" #include "utils/event_counter.hpp" #include "utils/file.hpp" @@ -62,7 +70,6 @@ #include "utils/settings.hpp" #include "utils/signals.hpp" #include "utils/string.hpp" -#include "utils/synchronized.hpp" #include "utils/sysinfo/memory.hpp" #include "utils/terminate_handler.hpp" #include "version.hpp" @@ -83,10 +90,10 @@ #include "communication/init.hpp" #include "communication/v2/server.hpp" #include "communication/v2/session.hpp" -#include "glue/communication.hpp" +#include "glue/v2/communication.hpp" #include "auth/auth.hpp" -#include "glue/auth.hpp" +#include "glue/v2/auth.hpp" #ifdef MG_ENTERPRISE #include "audit/log.hpp" @@ -197,16 +204,6 @@ DEFINE_bool(storage_wal_enabled, false, DEFINE_VALIDATED_uint64(storage_snapshot_retention_count, 3, "The number of snapshots that should always be kept.", FLAG_IN_RANGE(1, 1000000)); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -DEFINE_VALIDATED_uint64(storage_wal_file_size_kib, memgraph::storage::Config::Durability().wal_file_size_kibibytes, - "Minimum file size of each WAL file.", - FLAG_IN_RANGE(1, static_cast(1000) * 1024)); -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -DEFINE_VALIDATED_uint64(storage_wal_file_flush_every_n_tx, - memgraph::storage::Config::Durability().wal_file_flush_every_n_tx, - "Issue a 'fsync' call after this amount of transactions are written to the " - "WAL file. Set to 1 for fully synchronous operation.", - FLAG_IN_RANGE(1, 1000000)); -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DEFINE_bool(storage_snapshot_on_exit, false, "Controls whether the storage creates another snapshot on exit."); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) @@ -271,9 +268,9 @@ DEFINE_uint64( namespace { using namespace std::literals; inline constexpr std::array isolation_level_mappings{ - std::pair{"SNAPSHOT_ISOLATION"sv, memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION}, - std::pair{"READ_COMMITTED"sv, memgraph::storage::IsolationLevel::READ_COMMITTED}, - std::pair{"READ_UNCOMMITTED"sv, memgraph::storage::IsolationLevel::READ_UNCOMMITTED}}; + std::pair{"SNAPSHOT_ISOLATION"sv, memgraph::storage::v3::IsolationLevel::SNAPSHOT_ISOLATION}, + std::pair{"READ_COMMITTED"sv, memgraph::storage::v3::IsolationLevel::READ_COMMITTED}, + std::pair{"READ_UNCOMMITTED"sv, memgraph::storage::v3::IsolationLevel::READ_UNCOMMITTED}}; const std::string isolation_level_help_string = fmt::format("Default isolation level used for the transactions. Allowed values: {}", @@ -302,13 +299,6 @@ DEFINE_VALIDATED_string(isolation_level, "SNAPSHOT_ISOLATION", isolation_level_h }); namespace { -memgraph::storage::IsolationLevel ParseIsolationLevel() { - const auto isolation_level = - StringToEnum(FLAGS_isolation_level, isolation_level_mappings); - MG_ASSERT(isolation_level, "Invalid isolation level"); - return *isolation_level; -} - int64_t GetMemoryLimit() { if (FLAGS_memory_limit == 0) { auto maybe_total_memory = memgraph::utils::sysinfo::TotalMemory(); @@ -329,30 +319,6 @@ int64_t GetMemoryLimit() { } } // namespace -namespace { -std::vector query_modules_directories; -} // namespace -DEFINE_VALIDATED_string(query_modules_directory, "", - "Directory where modules with custom query procedures are stored. " - "NOTE: Multiple comma-separated directories can be defined.", - { - query_modules_directories.clear(); - if (value.empty()) return true; - const auto directories = memgraph::utils::Split(value, ","); - for (const auto &dir : directories) { - if (!memgraph::utils::DirExists(dir)) { - std::cout << "Expected --" << flagname << " to point to directories." << std::endl; - std::cout << dir << " is not a directory." << std::endl; - return false; - } - } - query_modules_directories.reserve(directories.size()); - std::transform(directories.begin(), directories.end(), - std::back_inserter(query_modules_directories), - [](const auto &dir) { return dir; }); - return true; - }); - // Logging flags DEFINE_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to logfiles"); DEFINE_string(log_file, "", "Path to where the log should be stored."); @@ -424,13 +390,6 @@ void InitializeLogger() { CreateLoggerFromSink(sinks, ParseLogLevel()); } -void AddLoggerSink(spdlog::sink_ptr new_sink) { - auto default_logger = spdlog::default_logger(); - auto sinks = default_logger->sinks(); - sinks.push_back(new_sink); - CreateLoggerFromSink(sinks, default_logger->level()); -} - } // namespace // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) @@ -443,431 +402,26 @@ DEFINE_string(organization_name, "", "Organization name."); struct SessionData { // Explicit constructor here to ensure that pointers to all objects are // supplied. -#if MG_ENTERPRISE - - SessionData(memgraph::storage::Storage *db, memgraph::query::InterpreterContext *interpreter_context, - memgraph::utils::Synchronized *auth, - memgraph::audit::Log *audit_log) - : db(db), interpreter_context(interpreter_context), auth(auth), audit_log(audit_log) {} - memgraph::storage::Storage *db; - memgraph::query::InterpreterContext *interpreter_context; - memgraph::utils::Synchronized *auth; - memgraph::audit::Log *audit_log; - -#else - - SessionData(memgraph::storage::Storage *db, memgraph::query::InterpreterContext *interpreter_context, - memgraph::utils::Synchronized *auth) - : db(db), interpreter_context(interpreter_context), auth(auth) {} - memgraph::storage::Storage *db; - memgraph::query::InterpreterContext *interpreter_context; - memgraph::utils::Synchronized *auth; - -#endif + SessionData(memgraph::coordinator::ShardMap &shard_map, memgraph::query::v2::InterpreterContext *interpreter_context) + : shard_map(&shard_map), interpreter_context(interpreter_context) {} + memgraph::coordinator::ShardMap *shard_map; + memgraph::query::v2::InterpreterContext *interpreter_context; }; inline constexpr std::string_view default_user_role_regex = "[a-zA-Z0-9_.+-@]+"; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -DEFINE_string(auth_user_or_role_name_regex, default_user_role_regex.data(), - "Set to the regular expression that each user or role name must fulfill."); - -class AuthQueryHandler final : public memgraph::query::AuthQueryHandler { - memgraph::utils::Synchronized *auth_; - std::string name_regex_string_; - std::regex name_regex_; - - public: - AuthQueryHandler(memgraph::utils::Synchronized *auth, - std::string name_regex_string) - : auth_(auth), name_regex_string_(std::move(name_regex_string)), name_regex_(name_regex_string_) {} - - bool CreateUser(const std::string &username, const std::optional &password) override { - if (name_regex_string_ != default_user_role_regex) { - if (const auto license_check_result = - memgraph::utils::license::global_license_checker.IsValidLicense(memgraph::utils::global_settings); - license_check_result.HasError()) { - throw memgraph::auth::AuthException( - "Custom user/role regex is a Memgraph Enterprise feature. Please set the config " - "(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}", - default_user_role_regex, - memgraph::utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex")); - } - } - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - try { - const auto [first_user, user_added] = std::invoke([&, this] { - auto locked_auth = auth_->Lock(); - const auto first_user = !locked_auth->HasUsers(); - const auto user_added = locked_auth->AddUser(username, password).has_value(); - return std::make_pair(first_user, user_added); - }); - - if (first_user) { - spdlog::info("{} is first created user. Granting all privileges.", username); - GrantPrivilege(username, memgraph::query::kPrivilegesAll); - } - - return user_added; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - bool DropUser(const std::string &username) override { - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - try { - auto locked_auth = auth_->Lock(); - auto user = locked_auth->GetUser(username); - if (!user) return false; - return locked_auth->RemoveUser(username); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - void SetPassword(const std::string &username, const std::optional &password) override { - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - try { - auto locked_auth = auth_->Lock(); - auto user = locked_auth->GetUser(username); - if (!user) { - throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist.", username); - } - user->UpdatePassword(password); - locked_auth->SaveUser(*user); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - bool CreateRole(const std::string &rolename) override { - if (!std::regex_match(rolename, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid role name."); - } - try { - auto locked_auth = auth_->Lock(); - return locked_auth->AddRole(rolename).has_value(); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - bool DropRole(const std::string &rolename) override { - if (!std::regex_match(rolename, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid role name."); - } - try { - auto locked_auth = auth_->Lock(); - auto role = locked_auth->GetRole(rolename); - if (!role) return false; - return locked_auth->RemoveRole(rolename); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - std::vector GetUsernames() override { - try { - auto locked_auth = auth_->ReadLock(); - std::vector usernames; - const auto &users = locked_auth->AllUsers(); - usernames.reserve(users.size()); - for (const auto &user : users) { - usernames.emplace_back(user.username()); - } - return usernames; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - std::vector GetRolenames() override { - try { - auto locked_auth = auth_->ReadLock(); - std::vector rolenames; - const auto &roles = locked_auth->AllRoles(); - rolenames.reserve(roles.size()); - for (const auto &role : roles) { - rolenames.emplace_back(role.rolename()); - } - return rolenames; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - std::optional GetRolenameForUser(const std::string &username) override { - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - try { - auto locked_auth = auth_->ReadLock(); - auto user = locked_auth->GetUser(username); - if (!user) { - throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username); - } - - if (const auto *role = user->role(); role != nullptr) { - return role->rolename(); - } - return std::nullopt; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - std::vector GetUsernamesForRole(const std::string &rolename) override { - if (!std::regex_match(rolename, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid role name."); - } - try { - auto locked_auth = auth_->ReadLock(); - auto role = locked_auth->GetRole(rolename); - if (!role) { - throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist.", rolename); - } - std::vector usernames; - const auto &users = locked_auth->AllUsersForRole(rolename); - usernames.reserve(users.size()); - for (const auto &user : users) { - usernames.emplace_back(user.username()); - } - return usernames; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - void SetRole(const std::string &username, const std::string &rolename) override { - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - if (!std::regex_match(rolename, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid role name."); - } - try { - auto locked_auth = auth_->Lock(); - auto user = locked_auth->GetUser(username); - if (!user) { - throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username); - } - auto role = locked_auth->GetRole(rolename); - if (!role) { - throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist .", rolename); - } - if (const auto *current_role = user->role(); current_role != nullptr) { - throw memgraph::query::QueryRuntimeException("User '{}' is already a member of role '{}'.", username, - current_role->rolename()); - } - user->SetRole(*role); - locked_auth->SaveUser(*user); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - void ClearRole(const std::string &username) override { - if (!std::regex_match(username, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user name."); - } - try { - auto locked_auth = auth_->Lock(); - auto user = locked_auth->GetUser(username); - if (!user) { - throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username); - } - user->ClearRole(); - locked_auth->SaveUser(*user); - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - std::vector> GetPrivileges(const std::string &user_or_role) override { - if (!std::regex_match(user_or_role, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user or role name."); - } - try { - auto locked_auth = auth_->ReadLock(); - std::vector> grants; - auto user = locked_auth->GetUser(user_or_role); - auto role = locked_auth->GetRole(user_or_role); - if (!user && !role) { - throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role); - } - if (user) { - const auto &permissions = user->GetPermissions(); - for (const auto &privilege : memgraph::query::kPrivilegesAll) { - auto permission = memgraph::glue::PrivilegeToPermission(privilege); - auto effective = permissions.Has(permission); - if (permissions.Has(permission) != memgraph::auth::PermissionLevel::NEUTRAL) { - std::vector description; - auto user_level = user->permissions().Has(permission); - if (user_level == memgraph::auth::PermissionLevel::GRANT) { - description.emplace_back("GRANTED TO USER"); - } else if (user_level == memgraph::auth::PermissionLevel::DENY) { - description.emplace_back("DENIED TO USER"); - } - if (const auto *role = user->role(); role != nullptr) { - auto role_level = role->permissions().Has(permission); - if (role_level == memgraph::auth::PermissionLevel::GRANT) { - description.emplace_back("GRANTED TO ROLE"); - } else if (role_level == memgraph::auth::PermissionLevel::DENY) { - description.emplace_back("DENIED TO ROLE"); - } - } - grants.push_back({memgraph::query::TypedValue(memgraph::auth::PermissionToString(permission)), - memgraph::query::TypedValue(memgraph::auth::PermissionLevelToString(effective)), - memgraph::query::TypedValue(memgraph::utils::Join(description, ", "))}); - } - } - } else { - const auto &permissions = role->permissions(); - for (const auto &privilege : memgraph::query::kPrivilegesAll) { - auto permission = memgraph::glue::PrivilegeToPermission(privilege); - auto effective = permissions.Has(permission); - if (effective != memgraph::auth::PermissionLevel::NEUTRAL) { - std::string description; - if (effective == memgraph::auth::PermissionLevel::GRANT) { - description = "GRANTED TO ROLE"; - } else if (effective == memgraph::auth::PermissionLevel::DENY) { - description = "DENIED TO ROLE"; - } - grants.push_back({memgraph::query::TypedValue(memgraph::auth::PermissionToString(permission)), - memgraph::query::TypedValue(memgraph::auth::PermissionLevelToString(effective)), - memgraph::query::TypedValue(description)}); - } - } - } - return grants; - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } - - void GrantPrivilege(const std::string &user_or_role, - const std::vector &privileges) override { - EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) { - // TODO (mferencevic): should we first check that the - // privilege is granted/denied/revoked before - // unconditionally granting/denying/revoking it? - permissions->Grant(permission); - }); - } - - void DenyPrivilege(const std::string &user_or_role, - const std::vector &privileges) override { - EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) { - // TODO (mferencevic): should we first check that the - // privilege is granted/denied/revoked before - // unconditionally granting/denying/revoking it? - permissions->Deny(permission); - }); - } - - void RevokePrivilege(const std::string &user_or_role, - const std::vector &privileges) override { - EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) { - // TODO (mferencevic): should we first check that the - // privilege is granted/denied/revoked before - // unconditionally granting/denying/revoking it? - permissions->Revoke(permission); - }); - } - - private: - template - void EditPermissions(const std::string &user_or_role, - const std::vector &privileges, const TEditFun &edit_fun) { - if (!std::regex_match(user_or_role, name_regex_)) { - throw memgraph::query::QueryRuntimeException("Invalid user or role name."); - } - try { - std::vector permissions; - permissions.reserve(privileges.size()); - for (const auto &privilege : privileges) { - permissions.push_back(memgraph::glue::PrivilegeToPermission(privilege)); - } - auto locked_auth = auth_->Lock(); - auto user = locked_auth->GetUser(user_or_role); - auto role = locked_auth->GetRole(user_or_role); - if (!user && !role) { - throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role); - } - if (user) { - for (const auto &permission : permissions) { - edit_fun(&user->permissions(), permission); - } - locked_auth->SaveUser(*user); - } else { - for (const auto &permission : permissions) { - edit_fun(&role->permissions(), permission); - } - locked_auth->SaveRole(*role); - } - } catch (const memgraph::auth::AuthException &e) { - throw memgraph::query::QueryRuntimeException(e.what()); - } - } -}; - -class AuthChecker final : public memgraph::query::AuthChecker { - public: - explicit AuthChecker( - memgraph::utils::Synchronized *auth) - : auth_{auth} {} - - static bool IsUserAuthorized(const memgraph::auth::User &user, - const std::vector &privileges) { - const auto user_permissions = user.GetPermissions(); - return std::all_of(privileges.begin(), privileges.end(), [&user_permissions](const auto privilege) { - return user_permissions.Has(memgraph::glue::PrivilegeToPermission(privilege)) == - memgraph::auth::PermissionLevel::GRANT; - }); - } - - bool IsUserAuthorized(const std::optional &username, - const std::vector &privileges) const final { - std::optional maybe_user; - { - auto locked_auth = auth_->ReadLock(); - if (!locked_auth->HasUsers()) { - return true; - } - if (username.has_value()) { - maybe_user = locked_auth->GetUser(*username); - } - } - - return maybe_user.has_value() && IsUserAuthorized(*maybe_user, privileges); - } - - private: - memgraph::utils::Synchronized *auth_; -}; class BoltSession final : public memgraph::communication::bolt::Session { public: - BoltSession(SessionData *data, const memgraph::communication::v2::ServerEndpoint &endpoint, + BoltSession(SessionData &data, const memgraph::communication::v2::ServerEndpoint &endpoint, memgraph::communication::v2::InputStream *input_stream, memgraph::communication::v2::OutputStream *output_stream) : memgraph::communication::bolt::Session(input_stream, output_stream), - db_(data->db), - interpreter_(data->interpreter_context), - auth_(data->auth), -#if MG_ENTERPRISE - audit_log_(data->audit_log), -#endif - endpoint_(endpoint) { - } + shard_map_(data.shard_map), + interpreter_(data.interpreter_context), + endpoint_(endpoint) {} using memgraph::communication::bolt::Session::TEncoder; @@ -880,29 +434,14 @@ class BoltSession final : public memgraph::communication::bolt::Session, std::optional> Interpret( const std::string &query, const std::map ¶ms) override { - std::map params_pv; - for (const auto &kv : params) params_pv.emplace(kv.first, memgraph::glue::ToPropertyValue(kv.second)); + std::map params_pv; + for (const auto &kv : params) params_pv.emplace(kv.first, memgraph::glue::v2::ToPropertyValue(kv.second)); const std::string *username{nullptr}; - if (user_) { - username = &user_->username(); - } -#ifdef MG_ENTERPRISE - if (memgraph::utils::license::global_license_checker.IsValidLicenseFast()) { - audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query, - memgraph::storage::PropertyValue(params_pv)); - } -#endif try { auto result = interpreter_.Prepare(query, params_pv, username); - if (user_ && !AuthChecker::IsUserAuthorized(*user_, result.privileges)) { - interpreter_.Abort(); - throw memgraph::communication::bolt::ClientError( - "You are not authorized to execute this query! Please contact " - "your database administrator."); - } return {result.headers, result.qid}; - } catch (const memgraph::query::QueryException &e) { + } catch (const memgraph::query::v2::QueryException &e) { // Wrap QueryException into ClientError, because we want to allow the // client to fix their query. throw memgraph::communication::bolt::ClientError(e.what()); @@ -911,26 +450,19 @@ class BoltSession final : public memgraph::communication::bolt::Session Pull(TEncoder *encoder, std::optional n, std::optional qid) override { - TypedValueResultStream stream(encoder, db_); + TypedValueResultStream stream(encoder, *shard_map_); return PullResults(stream, n, qid); } std::map Discard(std::optional n, std::optional qid) override { - memgraph::query::DiscardValueResultStream stream; + memgraph::query::v2::DiscardValueResultStream stream; return PullResults(stream, n, qid); } void Abort() override { interpreter_.Abort(); } - bool Authenticate(const std::string &username, const std::string &password) override { - auto locked_auth = auth_->Lock(); - if (!locked_auth->HasUsers()) { - return true; - } - user_ = locked_auth->Authenticate(username, password); - return user_.has_value(); - } + bool Authenticate(const std::string & /*username*/, const std::string & /*password*/) override { return true; } std::optional GetServerNameForInit() override { if (FLAGS_bolt_server_name_for_init.empty()) return std::nullopt; @@ -945,21 +477,21 @@ class BoltSession final : public memgraph::communication::bolt::Session decoded_summary; for (const auto &kv : summary) { - auto maybe_value = memgraph::glue::ToBoltValue(kv.second, *db_, memgraph::storage::View::NEW); + auto maybe_value = memgraph::glue::v2::ToBoltValue(kv.second, *shard_map_, memgraph::storage::v3::View::NEW); if (maybe_value.HasError()) { switch (maybe_value.GetError()) { - case memgraph::storage::Error::DELETED_OBJECT: - case memgraph::storage::Error::SERIALIZATION_ERROR: - case memgraph::storage::Error::VERTEX_HAS_EDGES: - case memgraph::storage::Error::PROPERTIES_DISABLED: - case memgraph::storage::Error::NONEXISTENT_OBJECT: + case memgraph::storage::v3::Error::DELETED_OBJECT: + case memgraph::storage::v3::Error::SERIALIZATION_ERROR: + case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: + case memgraph::storage::v3::Error::PROPERTIES_DISABLED: + case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming summary."); } } decoded_summary.emplace(kv.first, std::move(*maybe_value)); } return decoded_summary; - } catch (const memgraph::query::QueryException &e) { + } catch (const memgraph::query::v2::QueryException &e) { // Wrap QueryException into ClientError, because we want to allow the // client to fix their query. throw memgraph::communication::bolt::ClientError(e.what()); @@ -970,22 +502,23 @@ class BoltSession final : public memgraph::communication::bolt::Session &values) { + void Result(const std::vector &values) { std::vector decoded_values; decoded_values.reserve(values.size()); for (const auto &v : values) { - auto maybe_value = memgraph::glue::ToBoltValue(v, *db_, memgraph::storage::View::NEW); + auto maybe_value = memgraph::glue::v2::ToBoltValue(v, *shard_map_, memgraph::storage::v3::View::NEW); if (maybe_value.HasError()) { switch (maybe_value.GetError()) { - case memgraph::storage::Error::DELETED_OBJECT: + case memgraph::storage::v3::Error::DELETED_OBJECT: throw memgraph::communication::bolt::ClientError("Returning a deleted object as a result."); - case memgraph::storage::Error::NONEXISTENT_OBJECT: + case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: throw memgraph::communication::bolt::ClientError("Returning a nonexistent object as a result."); - case memgraph::storage::Error::VERTEX_HAS_EDGES: - case memgraph::storage::Error::SERIALIZATION_ERROR: - case memgraph::storage::Error::PROPERTIES_DISABLED: + case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: + case memgraph::storage::v3::Error::SERIALIZATION_ERROR: + case memgraph::storage::v3::Error::PROPERTIES_DISABLED: throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming results."); } } @@ -997,17 +530,12 @@ class BoltSession final : public memgraph::communication::bolt::Session *auth_; - std::optional user_; -#ifdef MG_ENTERPRISE - memgraph::audit::Log *audit_log_; -#endif + const memgraph::coordinator::ShardMap *shard_map_; + memgraph::query::v2::Interpreter interpreter_; memgraph::communication::v2::ServerEndpoint endpoint_; }; @@ -1059,80 +587,12 @@ int main(int argc, char **argv) { // Unhandled exception handler init. std::set_terminate(&memgraph::utils::TerminateHandler); - // Initialize Python - auto *program_name = Py_DecodeLocale(argv[0], nullptr); - MG_ASSERT(program_name); - // Set program name, so Python can find its way to runtime libraries relative - // to executable. - Py_SetProgramName(program_name); - PyImport_AppendInittab("_mgp", &memgraph::query::procedure::PyInitMgpModule); - Py_InitializeEx(0 /* = initsigs */); - PyEval_InitThreads(); - Py_BEGIN_ALLOW_THREADS; - - // Add our Python modules to sys.path - try { - auto exe_path = memgraph::utils::GetExecutablePath(); - auto py_support_dir = exe_path.parent_path() / "python_support"; - if (std::filesystem::is_directory(py_support_dir)) { - auto gil = memgraph::py::EnsureGIL(); - auto maybe_exc = memgraph::py::AppendToSysPath(py_support_dir.c_str()); - if (maybe_exc) { - spdlog::error(memgraph::utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc, - "https://memgr.ph/python")); - } else { - // Change how we load dynamic libraries on Python by using RTLD_NOW and - // RTLD_DEEPBIND flags. This solves an issue with using the wrong version of - // libstd. - auto gil = memgraph::py::EnsureGIL(); - // NOLINTNEXTLINE(hicpp-signed-bitwise) - auto *flag = PyLong_FromLong(RTLD_NOW | RTLD_DEEPBIND); - auto *setdl = PySys_GetObject("setdlopenflags"); - MG_ASSERT(setdl); - auto *arg = PyTuple_New(1); - MG_ASSERT(arg); - MG_ASSERT(PyTuple_SetItem(arg, 0, flag) == 0); - PyObject_CallObject(setdl, arg); - Py_DECREF(flag); - Py_DECREF(setdl); - Py_DECREF(arg); - } - } else { - spdlog::error( - memgraph::utils::MessageWithLink("Unable to load support for embedded Python: missing directory {}.", - py_support_dir, "https://memgr.ph/python")); - } - } catch (const std::filesystem::filesystem_error &e) { - spdlog::error(memgraph::utils::MessageWithLink("Unable to load support for embedded Python: {}.", e.what(), - "https://memgr.ph/python")); - } - // Initialize the communication library. memgraph::communication::SSLInit sslInit; // Initialize the requests library. memgraph::requests::Init(); - // Start memory warning logger. - memgraph::utils::Scheduler mem_log_scheduler; - if (FLAGS_memory_warning_threshold > 0) { - auto free_ram = memgraph::utils::sysinfo::AvailableMemory(); - if (free_ram) { - mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] { - auto free_ram = memgraph::utils::sysinfo::AvailableMemory(); - if (free_ram && *free_ram / 1024 < FLAGS_memory_warning_threshold) - spdlog::warn(memgraph::utils::MessageWithLink("Running out of available RAM, only {} MB left.", - *free_ram / 1024, "https://memgr.ph/ram")); - }); - } else { - // Kernel version for the `MemAvailable` value is from: man procfs - spdlog::warn( - "You have an older kernel version (<3.14) or the /proc " - "filesystem isn't available so remaining memory warnings " - "won't be available."); - } - } - std::cout << "You are running Memgraph v" << gflags::VersionString() << std::endl; std::cout << "To get started with Memgraph, visit https://memgr.ph/start" << std::endl; @@ -1165,66 +625,35 @@ int main(int argc, char **argv) { // Example: When the main storage is destructed it makes a snapshot. When // audit logging is destructed it syncs all pending data to disk and that can // fail. That is why it must be destructed *after* the main database storage - // to minimise the impact of their failure on the main storage. + // to minimize the impact of their failure on the main storage. - // Begin enterprise features initialization + memgraph::io::local_transport::LocalSystem ls; + auto unique_local_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); + auto io = ls.Register(unique_local_addr_query); - // Auth - memgraph::utils::Synchronized auth{data_directory / - "auth"}; + memgraph::machine_manager::MachineConfig config{ + .coordinator_addresses = std::vector{unique_local_addr_query}, + .is_storage = true, + .is_coordinator = true, + .listen_ip = unique_local_addr_query.last_known_ip, + .listen_port = unique_local_addr_query.last_known_port, + }; -#ifdef MG_ENTERPRISE - // Audit log - memgraph::audit::Log audit_log{data_directory / "audit", FLAGS_audit_buffer_size, - FLAGS_audit_buffer_flush_interval_ms}; - // Start the log if enabled. - if (FLAGS_audit_enabled) { - audit_log.Start(); - } - // Setup SIGUSR2 to be used for reopening audit log files, when e.g. logrotate - // rotates our audit logs. - MG_ASSERT(memgraph::utils::SignalHandler::RegisterHandler(memgraph::utils::Signal::User2, - [&audit_log]() { audit_log.ReopenLog(); }), - "Unable to register SIGUSR2 handler!"); + memgraph::coordinator::ShardMap sm; + auto prop_map = sm.AllocatePropertyIds(std::vector{"property"}); + auto edge_type_map = sm.AllocateEdgeTypeIds(std::vector{"edge_type"}); + std::vector schema{ + {prop_map.at("property"), memgraph::common::SchemaType::INT}}; + sm.InitializeNewLabel("label", schema, 1, sm.shard_map_version); - // End enterprise features initialization -#endif + memgraph::coordinator::Coordinator coordinator{sm}; - // Main storage and execution engines initialization - memgraph::storage::Config db_config{ - .gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC, - .interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)}, - .items = {.properties_on_edges = FLAGS_storage_properties_on_edges}, - .durability = {.storage_directory = FLAGS_data_directory, - .recover_on_startup = FLAGS_storage_recover_on_startup, - .snapshot_retention_count = FLAGS_storage_snapshot_retention_count, - .wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib, - .wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx, - .snapshot_on_exit = FLAGS_storage_snapshot_on_exit, - .restore_replicas_on_startup = FLAGS_storage_restore_replicas_on_startup}, - .transaction = {.isolation_level = ParseIsolationLevel()}}; - if (FLAGS_storage_snapshot_interval_sec == 0) { - if (FLAGS_storage_wal_enabled) { - LOG_FATAL( - "In order to use write-ahead-logging you must enable " - "periodic snapshots by setting the snapshot interval to a " - "value larger than 0!"); - db_config.durability.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED; - } - } else { - if (FLAGS_storage_wal_enabled) { - db_config.durability.snapshot_wal_mode = - memgraph::storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL; - } else { - db_config.durability.snapshot_wal_mode = - memgraph::storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT; - } - db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec); - } - memgraph::storage::Storage db(db_config); + memgraph::machine_manager::MachineManager mm{io, config, coordinator, + sm}; + std::jthread mm_thread([&mm] { mm.Run(); }); - memgraph::query::InterpreterContext interpreter_context{ - &db, + memgraph::query::v2::InterpreterContext interpreter_context{ + (memgraph::storage::v3::Shard *)(nullptr), {.query = {.allow_load_csv = FLAGS_allow_load_csv}, .execution_timeout_sec = FLAGS_query_execution_timeout_sec, .replication_replica_check_frequency = std::chrono::seconds(FLAGS_replication_replica_check_frequency_sec), @@ -1232,32 +661,14 @@ int main(int argc, char **argv) { .default_pulsar_service_url = FLAGS_pulsar_service_url, .stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries, .stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)}, - FLAGS_data_directory}; -#ifdef MG_ENTERPRISE - SessionData session_data{&db, &interpreter_context, &auth, &audit_log}; -#else - SessionData session_data{&db, &interpreter_context, &auth}; -#endif + FLAGS_data_directory, + std::move(io), + mm.CoordinatorAddress()}; - memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories, FLAGS_data_directory); - memgraph::query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories(); + SessionData session_data{sm, &interpreter_context}; - AuthQueryHandler auth_handler(&auth, FLAGS_auth_user_or_role_name_regex); - AuthChecker auth_checker{&auth}; - interpreter_context.auth = &auth_handler; - interpreter_context.auth_checker = &auth_checker; - - { - // Triggers can execute query procedures, so we need to reload the modules first and then - // the triggers - auto storage_accessor = interpreter_context.db->Access(); - auto dba = memgraph::query::DbAccessor{&storage_accessor}; - interpreter_context.trigger_store.RestoreTriggers( - &interpreter_context.ast_cache, &dba, interpreter_context.config.query, interpreter_context.auth_checker); - } - - // As the Stream transformations are using modules, they have to be restored after the query modules are loaded. - interpreter_context.streams.RestoreStreams(); + interpreter_context.auth = nullptr; + interpreter_context.auth_checker = nullptr; ServerContext context; std::string service_name = "Bolt"; @@ -1272,61 +683,27 @@ int main(int argc, char **argv) { auto server_endpoint = memgraph::communication::v2::ServerEndpoint{ boost::asio::ip::address::from_string(FLAGS_bolt_address), static_cast(FLAGS_bolt_port)}; - ServerT server(server_endpoint, &session_data, &context, FLAGS_bolt_session_inactivity_timeout, service_name, + ServerT server(server_endpoint, session_data, &context, FLAGS_bolt_session_inactivity_timeout, service_name, FLAGS_bolt_num_workers); // Setup telemetry std::optional telemetry; - if (FLAGS_telemetry_enabled) { - telemetry.emplace("https://telemetry.memgraph.com/88b5e7e8-746a-11e8-9f85-538a9e9690cc/", - data_directory / "telemetry", std::chrono::minutes(10)); - telemetry->AddCollector("storage", [&db]() -> nlohmann::json { - auto info = db.GetInfo(); - return {{"vertices", info.vertex_count}, {"edges", info.edge_count}}; - }); - telemetry->AddCollector("event_counters", []() -> nlohmann::json { - nlohmann::json ret; - for (size_t i = 0; i < EventCounter::End(); ++i) { - ret[EventCounter::GetName(i)] = EventCounter::global_counters[i].load(std::memory_order_relaxed); - } - return ret; - }); - telemetry->AddCollector("query_module_counters", []() -> nlohmann::json { - return memgraph::query::plan::CallProcedure::GetAndResetCounters(); - }); - } - - memgraph::communication::websocket::SafeAuth websocket_auth{&auth}; - memgraph::communication::websocket::Server websocket_server{ - {FLAGS_monitoring_address, static_cast(FLAGS_monitoring_port)}, &context, websocket_auth}; - AddLoggerSink(websocket_server.GetLoggingSink()); // Handler for regular termination signals - auto shutdown = [&websocket_server, &server, &interpreter_context] { + auto shutdown = [&server, &interpreter_context] { // Server needs to be shutdown first and then the database. This prevents // a race condition when a transaction is accepted during server shutdown. server.Shutdown(); // After the server is notified to stop accepting and processing // connections we tell the execution engine to stop processing all pending // queries. - memgraph::query::Shutdown(&interpreter_context); - websocket_server.Shutdown(); + memgraph::query::v2::Shutdown(&interpreter_context); }; InitSignalHandlers(shutdown); MG_ASSERT(server.Start(), "Couldn't start the Bolt server!"); - websocket_server.Start(); - server.AwaitShutdown(); - websocket_server.AwaitShutdown(); - - memgraph::query::procedure::gModuleRegistry.UnloadAllModules(); - - Py_END_ALLOW_THREADS; - // Shutdown Python - Py_Finalize(); - PyMem_RawFree(program_name); memgraph::utils::total_memory_tracker.LogPeakMemoryUsage(); return 0; diff --git a/src/query/v2/CMakeLists.txt b/src/query/v2/CMakeLists.txt index ebe7e41e4..03167341a 100644 --- a/src/query/v2/CMakeLists.txt +++ b/src/query/v2/CMakeLists.txt @@ -9,7 +9,6 @@ set(mg_query_v2_sources ${lcp_query_v2_cpp_files} common.cpp cypher_query_interpreter.cpp - dump.cpp frontend/semantic/required_privileges.cpp frontend/stripped.cpp interpret/awesome_memgraph_functions.cpp @@ -23,16 +22,7 @@ set(mg_query_v2_sources plan/rewrite/index_lookup.cpp plan/rule_based_planner.cpp plan/variable_start_planner.cpp - # procedure/mg_procedure_impl.cpp - # procedure/mg_procedure_helpers.cpp - # procedure/module.cpp - # procedure/py_module.cpp serialization/property_value.cpp - # stream/streams.cpp - # stream/sources.cpp - # stream/common.cpp - # trigger.cpp - # trigger_context.cpp bindings/typed_value.cpp accessors.cpp) diff --git a/src/query/v2/accessors.cpp b/src/query/v2/accessors.cpp index 1fd4a3d91..87fb74aa3 100644 --- a/src/query/v2/accessors.cpp +++ b/src/query/v2/accessors.cpp @@ -11,12 +11,13 @@ #include "query/v2/accessors.hpp" #include "query/v2/requests.hpp" +#include "storage/v3/id_types.hpp" namespace memgraph::query::v2::accessors { EdgeAccessor::EdgeAccessor(Edge edge, std::vector> props) : edge(std::move(edge)), properties(std::move(props)) {} -uint64_t EdgeAccessor::EdgeType() const { return edge.type.id; } +EdgeTypeId EdgeAccessor::EdgeType() const { return EdgeTypeId::FromUint(edge.type.id); } std::vector> EdgeAccessor::Properties() const { return properties; diff --git a/src/query/v2/accessors.hpp b/src/query/v2/accessors.hpp index 1d545650f..e1a20ee07 100644 --- a/src/query/v2/accessors.hpp +++ b/src/query/v2/accessors.hpp @@ -30,6 +30,7 @@ using Edge = memgraph::msgs::Edge; using Vertex = memgraph::msgs::Vertex; using Label = memgraph::msgs::Label; using PropertyId = memgraph::msgs::PropertyId; +using EdgeTypeId = memgraph::msgs::EdgeTypeId; class VertexAccessor; @@ -37,7 +38,7 @@ class EdgeAccessor final { public: EdgeAccessor(Edge edge, std::vector> props); - uint64_t EdgeType() const; + EdgeTypeId EdgeType() const; std::vector> Properties() const; diff --git a/src/query/v2/common.hpp b/src/query/v2/common.hpp index dceeb51a5..5637a27bc 100644 --- a/src/query/v2/common.hpp +++ b/src/query/v2/common.hpp @@ -100,41 +100,6 @@ template concept RecordAccessor = AccessorWithSetProperty || AccessorWithSetPropertyAndValidate; -inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) { - switch (schema_violation.status) { - case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: { - throw SchemaViolationException( - fmt::format("Primary key {} not defined on label :{}", - storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type), - dba.LabelToName(schema_violation.label))); - } - case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: { - throw SchemaViolationException( - fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label))); - } - case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: { - throw SchemaViolationException( - fmt::format("Wrong type of property {} in schema :{}, should be of type {}", - *schema_violation.violated_property_value, dba.LabelToName(schema_violation.label), - storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type))); - } - case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: { - throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported", - *schema_violation.violated_property_value, - dba.LabelToName(schema_violation.label))); - } - case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: { - throw SchemaViolationException(fmt::format( - "Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value, - dba.LabelToName(schema_violation.label))); - } - case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: { - throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}", - dba.LabelToName(schema_violation.label))); - } - } -} - inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) { switch (error) { case storage::v3::Error::SERIALIZATION_ERROR: @@ -149,35 +114,5 @@ inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) { } } -/// Set a property `value` mapped with given `key` on a `record`. -/// -/// @throw QueryRuntimeException if value cannot be set as a property value -template -storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key, - const TypedValue &value) { - try { - if constexpr (std::is_same_v) { - const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::TypedToPropertyValue(value)); - if (maybe_old_value.HasError()) { - std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); }, - [&dba](const storage::v3::SchemaViolation &schema_violation) { - HandleSchemaViolation(schema_violation, dba); - }}, - maybe_old_value.GetError()); - } - return std::move(*maybe_old_value); - } else { - // No validation on edge properties - const auto maybe_old_value = record->SetProperty(key, storage::v3::TypedToPropertyValue(value)); - if (maybe_old_value.HasError()) { - HandleErrorOnPropertyUpdate(maybe_old_value.GetError()); - } - return std::move(*maybe_old_value); - } - } catch (const expr::TypedValueException &) { - throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type()); - } -} - int64_t QueryTimestamp(); } // namespace memgraph::query::v2 diff --git a/src/query/v2/context.hpp b/src/query/v2/context.hpp index 48ee4c923..3122b3769 100644 --- a/src/query/v2/context.hpp +++ b/src/query/v2/context.hpp @@ -13,6 +13,7 @@ #include +#include "io/local_transport/local_transport.hpp" #include "query/v2/bindings/symbol_table.hpp" #include "query/v2/common.hpp" #include "query/v2/metadata.hpp" @@ -42,21 +43,28 @@ struct EvaluationContext { mutable std::unordered_map counters; }; -inline std::vector NamesToProperties(const std::vector &property_names, - DbAccessor *dba) { +inline std::vector NamesToProperties( + const std::vector &property_names, msgs::ShardRequestManagerInterface *shard_request_manager) { std::vector properties; + // TODO Fix by using reference properties.reserve(property_names.size()); - for (const auto &name : property_names) { - properties.push_back(dba->NameToProperty(name)); + if (shard_request_manager != nullptr) { + for (const auto &name : property_names) { + properties.push_back(shard_request_manager->NameToProperty(name)); + } } return properties; } -inline std::vector NamesToLabels(const std::vector &label_names, DbAccessor *dba) { +inline std::vector NamesToLabels(const std::vector &label_names, + msgs::ShardRequestManagerInterface *shard_request_manager) { std::vector labels; labels.reserve(label_names.size()); - for (const auto &name : label_names) { - labels.push_back(dba->NameToLabel(name)); + // TODO Fix by using reference + if (shard_request_manager != nullptr) { + for (const auto &name : label_names) { + labels.push_back(shard_request_manager->LabelNameToLabelId(name)); + } } return labels; } @@ -73,7 +81,7 @@ struct ExecutionContext { ExecutionStats execution_stats; // TriggerContextCollector *trigger_context_collector{nullptr}; utils::AsyncTimer timer; - std::unique_ptr shard_request_manager{nullptr}; + msgs::ShardRequestManagerInterface *shard_request_manager{nullptr}; }; static_assert(std::is_move_assignable_v, "ExecutionContext must be move assignable!"); diff --git a/src/query/v2/cypher_query_interpreter.cpp b/src/query/v2/cypher_query_interpreter.cpp index 96e653b06..d365a53fb 100644 --- a/src/query/v2/cypher_query_interpreter.cpp +++ b/src/query/v2/cypher_query_interpreter.cpp @@ -11,6 +11,7 @@ #include "query/v2/cypher_query_interpreter.hpp" #include "query/v2/bindings/symbol_generator.hpp" +#include "query/v2/shard_request_manager.hpp" // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner."); @@ -117,9 +118,9 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, - DbAccessor *db_accessor, + msgs::ShardRequestManagerInterface *shard_manager, const std::vector &predefined_identifiers) { - auto vertex_counts = plan::MakeVertexCountCache(db_accessor); + auto vertex_counts = plan::MakeVertexCountCache(shard_manager); auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers); auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts); auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner); @@ -129,7 +130,7 @@ std::unique_ptr MakeLogicalPlan(AstStorage ast_storage, CypherQuery std::shared_ptr CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, utils::SkipList *plan_cache, - DbAccessor *db_accessor, + msgs::ShardRequestManagerInterface *shard_manager, const std::vector &predefined_identifiers) { std::optional::Accessor> plan_cache_access; if (plan_cache) { @@ -145,7 +146,7 @@ std::shared_ptr CypherQueryToPlan(uint64_t hash, AstStorage ast_stor } auto plan = std::make_shared( - MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers)); + MakeLogicalPlan(std::move(ast_storage), query, parameters, shard_manager, predefined_identifiers)); if (plan_cache_access) { plan_cache_access->insert({hash, plan}); } diff --git a/src/query/v2/cypher_query_interpreter.hpp b/src/query/v2/cypher_query_interpreter.hpp index a2c7223f2..74dbe85d5 100644 --- a/src/query/v2/cypher_query_interpreter.hpp +++ b/src/query/v2/cypher_query_interpreter.hpp @@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan { }; std::unique_ptr MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, - DbAccessor *db_accessor, + msgs::ShardRequestManagerInterface *shard_manager, const std::vector &predefined_identifiers); /** @@ -145,7 +145,7 @@ std::unique_ptr MakeLogicalPlan(AstStorage ast_storage, CypherQuery */ std::shared_ptr CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, utils::SkipList *plan_cache, - DbAccessor *db_accessor, + msgs::ShardRequestManagerInterface *shard_manager, const std::vector &predefined_identifiers = {}); } // namespace memgraph::query::v2 diff --git a/src/query/v2/db_accessor.hpp b/src/query/v2/db_accessor.hpp index a43cdf18f..a975a70ce 100644 --- a/src/query/v2/db_accessor.hpp +++ b/src/query/v2/db_accessor.hpp @@ -219,195 +219,7 @@ inline VertexAccessor EdgeAccessor::From() const { return *static_cast FindVertex(uint64_t /*unused*/) { return std::nullopt; } - - std::optional FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) { - auto maybe_vertex = accessor_->FindVertex(primary_key, view); - if (maybe_vertex) return VertexAccessor(*maybe_vertex); - return std::nullopt; - } - - VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); } - - VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) { - return VerticesIterable(accessor_->Vertices(label, view)); - } - - VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) { - return VerticesIterable(accessor_->Vertices(label, property, view)); - } - - VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property, - const storage::v3::PropertyValue &value) { - return VerticesIterable(accessor_->Vertices(label, property, value, view)); - } - - VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property, - const std::optional> &lower, - const std::optional> &upper) { - return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view)); - } - - storage::v3::ResultSchema InsertVertexAndValidate( - const storage::v3::LabelId primary_label, const std::vector &labels, - const std::vector> &properties) { - auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties); - if (maybe_vertex_acc.HasError()) { - return {std::move(maybe_vertex_acc.GetError())}; - } - return VertexAccessor{maybe_vertex_acc.GetValue()}; - } - - storage::v3::Result InsertEdge(VertexAccessor *from, VertexAccessor *to, - const storage::v3::EdgeTypeId &edge_type) { - static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0); - auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(), - to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid); - if (maybe_edge.HasError()) return storage::v3::Result(maybe_edge.GetError()); - return EdgeAccessor(*maybe_edge); - } - - storage::v3::Result> RemoveEdge(EdgeAccessor *edge) { - auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid()); - if (res.HasError()) { - return res.GetError(); - } - - const auto &value = res.GetValue(); - if (!value) { - return std::optional{}; - } - - return std::make_optional(*value); - } - - storage::v3::Result>>> DetachRemoveVertex( - VertexAccessor *vertex_accessor) { - using ReturnType = std::pair>; - - auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_); - if (res.HasError()) { - return res.GetError(); - } - - const auto &value = res.GetValue(); - if (!value) { - return std::optional{}; - } - - const auto &[vertex, edges] = *value; - - std::vector deleted_edges; - deleted_edges.reserve(edges.size()); - std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges), - [](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; }); - - return std::make_optional(vertex, std::move(deleted_edges)); - } - - storage::v3::Result> RemoveVertex(VertexAccessor *vertex_accessor) { - auto res = accessor_->DeleteVertex(&vertex_accessor->impl_); - if (res.HasError()) { - return res.GetError(); - } - - const auto &value = res.GetValue(); - if (!value) { - return std::optional{}; - } - - return {std::make_optional(*value)}; - } - - // TODO(jbajic) Query engine should have a map of labels, properties and edge - // types - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); } - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); } - - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); } - - const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); } - - const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); } - - const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); } - - void AdvanceCommand() { accessor_->AdvanceCommand(); } - - void Commit() { return accessor_->Commit(coordinator::Hlc{}); } - - void Abort() { accessor_->Abort(); } - - bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); } - - bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const { - return accessor_->LabelPropertyIndexExists(label, prop); - } - - int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); } - - int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); } - - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const { - return accessor_->ApproximateVertexCount(label, property); - } - - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, - const storage::v3::PropertyValue &value) const { - return accessor_->ApproximateVertexCount(label, property, value); - } - - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, - const std::optional> &lower, - const std::optional> &upper) const { - return accessor_->ApproximateVertexCount(label, property, lower, upper); - } - - storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); } - - const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); } - - storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); } }; } // namespace memgraph::query::v2 diff --git a/src/query/v2/dump.cpp b/src/query/v2/dump.cpp deleted file mode 100644 index bf20abd0d..000000000 --- a/src/query/v2/dump.cpp +++ /dev/null @@ -1,483 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/dump.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/db_accessor.hpp" -#include "query/v2/exceptions.hpp" -#include "query/v2/stream.hpp" -#include "storage/v3/property_value.hpp" -#include "storage/v3/storage.hpp" -#include "utils/algorithm.hpp" -#include "utils/logging.hpp" -#include "utils/string.hpp" -#include "utils/temporal.hpp" - -namespace memgraph::query::v2 { - -namespace { - -// Property that is used to make a difference among vertices. It is added to -// property set of vertices to match edges and removed after the entire graph -// is built. -const char *kInternalPropertyId = "__mg_id__"; - -// Label that is attached to each vertex and is used for easier creation of -// index on internal property id. -const char *kInternalVertexLabel = "__mg_vertex__"; - -/// A helper function that escapes label, edge type and property names. -std::string EscapeName(const std::string_view value) { - std::string out; - out.reserve(value.size() + 2); - out.append(1, '`'); - for (auto c : value) { - if (c == '`') { - out.append("``"); - } else { - out.append(1, c); - } - } - out.append(1, '`'); - return out; -} - -void DumpPreciseDouble(std::ostream *os, double value) { - // A temporary stream is used to keep precision of the original output - // stream unchanged. - std::ostringstream temp_oss; - temp_oss << std::setprecision(std::numeric_limits::max_digits10) << value; - *os << temp_oss.str(); -} - -namespace { -void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) { - utils::Date date(value.microseconds); - os << "DATE(\"" << date << "\")"; -} - -void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) { - utils::LocalTime lt(value.microseconds); - os << "LOCALTIME(\"" << lt << "\")"; -} - -void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) { - utils::LocalDateTime ldt(value.microseconds); - os << "LOCALDATETIME(\"" << ldt << "\")"; -} - -void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) { - utils::Duration dur(value.microseconds); - os << "DURATION(\"" << dur << "\")"; -} - -void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) { - switch (value.type) { - case storage::v3::TemporalType::Date: { - DumpDate(os, value); - return; - } - case storage::v3::TemporalType::LocalTime: { - DumpLocalTime(os, value); - return; - } - case storage::v3::TemporalType::LocalDateTime: { - DumpLocalDateTime(os, value); - return; - } - case storage::v3::TemporalType::Duration: { - DumpDuration(os, value); - return; - } - } -} -} // namespace - -void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) { - switch (value.type()) { - case storage::v3::PropertyValue::Type::Null: - *os << "Null"; - return; - case storage::v3::PropertyValue::Type::Bool: - *os << (value.ValueBool() ? "true" : "false"); - return; - case storage::v3::PropertyValue::Type::String: - *os << utils::Escape(value.ValueString()); - return; - case storage::v3::PropertyValue::Type::Int: - *os << value.ValueInt(); - return; - case storage::v3::PropertyValue::Type::Double: - DumpPreciseDouble(os, value.ValueDouble()); - return; - case storage::v3::PropertyValue::Type::List: { - *os << "["; - const auto &list = value.ValueList(); - utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); }); - *os << "]"; - return; - } - case storage::v3::PropertyValue::Type::Map: { - *os << "{"; - const auto &map = value.ValueMap(); - utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) { - os << EscapeName(kv.first) << ": "; - DumpPropertyValue(&os, kv.second); - }); - *os << "}"; - return; - } - case storage::v3::PropertyValue::Type::TemporalData: { - DumpTemporalData(*os, value.ValueTemporalData()); - return; - } - } -} - -void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba, - const std::map &store, - std::optional property_id = std::nullopt) { - *os << "{"; - if (property_id) { - *os << kInternalPropertyId << ": " << *property_id; - if (store.size() > 0) *os << ", "; - } - utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) { - os << EscapeName(dba->PropertyToName(kv.first)) << ": "; - DumpPropertyValue(&os, kv.second); - }); - *os << "}"; -} - -void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) { - *os << "CREATE ("; - *os << ":" << kInternalVertexLabel; - auto maybe_labels = vertex.Labels(storage::v3::View::OLD); - if (maybe_labels.HasError()) { - switch (maybe_labels.GetError()) { - case storage::v3::Error::DELETED_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node."); - case storage::v3::Error::NONEXISTENT_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist."); - case storage::v3::Error::SERIALIZATION_ERROR: - case storage::v3::Error::VERTEX_HAS_EDGES: - case storage::v3::Error::PROPERTIES_DISABLED: - throw query::v2::QueryRuntimeException("Unexpected error when getting labels."); - } - } - for (const auto &label : *maybe_labels) { - *os << ":" << EscapeName(dba->LabelToName(label)); - } - *os << " "; - auto maybe_props = vertex.Properties(storage::v3::View::OLD); - if (maybe_props.HasError()) { - switch (maybe_props.GetError()) { - case storage::v3::Error::DELETED_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object."); - case storage::v3::Error::NONEXISTENT_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist."); - case storage::v3::Error::SERIALIZATION_ERROR: - case storage::v3::Error::VERTEX_HAS_EDGES: - case storage::v3::Error::PROPERTIES_DISABLED: - throw query::v2::QueryRuntimeException("Unexpected error when getting properties."); - } - } - DumpProperties(os, dba, *maybe_props, vertex.CypherId()); - *os << ");"; -} - -void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) { - *os << "MATCH "; - *os << "(u:" << kInternalVertexLabel << "), "; - *os << "(v:" << kInternalVertexLabel << ")"; - *os << " WHERE "; - *os << "u." << kInternalPropertyId << " = " << edge.From().CypherId(); - *os << " AND "; - *os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " "; - *os << "CREATE (u)-["; - *os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType())); - auto maybe_props = edge.Properties(storage::v3::View::OLD); - if (maybe_props.HasError()) { - switch (maybe_props.GetError()) { - case storage::v3::Error::DELETED_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object."); - case storage::v3::Error::NONEXISTENT_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist."); - case storage::v3::Error::SERIALIZATION_ERROR: - case storage::v3::Error::VERTEX_HAS_EDGES: - case storage::v3::Error::PROPERTIES_DISABLED: - throw query::v2::QueryRuntimeException("Unexpected error when getting properties."); - } - } - if (maybe_props->size() > 0) { - *os << " "; - DumpProperties(os, dba, *maybe_props); - } - *os << "]->(v);"; -} - -void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) { - *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";"; -} - -void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, - storage::v3::PropertyId property) { - *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property)) - << ");"; -} - -void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, - storage::v3::PropertyId property) { - *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u." - << EscapeName(dba->PropertyToName(property)) << ");"; -} - -void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, - const std::set &properties) { - *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT "; - utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) { - stream << "u." << EscapeName(dba->PropertyToName(property)); - }); - *os << " IS UNIQUE;"; -} - -} // namespace - -PullPlanDump::PullPlanDump(DbAccessor *dba) - : dba_(dba), - vertices_iterable_(dba->Vertices(storage::v3::View::OLD)), - pull_chunks_{// Dump all label indices - CreateLabelIndicesPullChunk(), - // Dump all label property indices - CreateLabelPropertyIndicesPullChunk(), - // Create internal index for faster edge creation - CreateInternalIndexPullChunk(), - // Dump all vertices - CreateVertexPullChunk(), - // Dump all edges - CreateEdgePullChunk(), - // Drop the internal index - CreateDropInternalIndexPullChunk(), - // Internal index cleanup - CreateInternalIndexCleanupPullChunk()} {} - -bool PullPlanDump::Pull(AnyStream *stream, std::optional n) { - // Iterate all functions that stream some results. - // Each function should return number of results it streamed after it - // finishes. If the function did not finish streaming all the results, - // std::nullopt should be returned because n results have already been sent. - while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) { - const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n); - - if (!maybe_streamed_count) { - // n wasn't large enough to stream all the results from the current chunk - break; - } - - if (n) { - // chunk finished streaming its results - // subtract number of results streamed in current pull - // so we know how many results we need to stream from future - // chunks. - *n -= *maybe_streamed_count; - } - - ++current_chunk_index_; - } - return current_chunk_index_ == pull_chunks_.size(); -} - -PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() { - // Dump all label indices - return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the construction of indices vectors - if (!indices_info_) { - indices_info_.emplace(dba_->ListAllIndices()); - } - const auto &label = indices_info_->label; - - size_t local_counter = 0; - while (global_index < label.size() && (!n || local_counter < *n)) { - std::ostringstream os; - DumpLabelIndex(&os, dba_, label[global_index]); - stream->Result({TypedValue(os.str())}); - - ++global_index; - ++local_counter; - } - - if (global_index == label.size()) { - return local_counter; - } - - return std::nullopt; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() { - return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the construction of indices vectors - if (!indices_info_) { - indices_info_.emplace(dba_->ListAllIndices()); - } - const auto &label_property = indices_info_->label_property; - - size_t local_counter = 0; - while (global_index < label_property.size() && (!n || local_counter < *n)) { - std::ostringstream os; - const auto &label_property_index = label_property[global_index]; - DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second); - stream->Result({TypedValue(os.str())}); - - ++global_index; - ++local_counter; - } - - if (global_index == label_property.size()) { - return local_counter; - } - - return std::nullopt; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() { - return [this](AnyStream *stream, std::optional) mutable -> std::optional { - if (vertices_iterable_.begin() != vertices_iterable_.end()) { - std::ostringstream os; - os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");"; - stream->Result({TypedValue(os.str())}); - internal_index_created_ = true; - return 1; - } - return 0; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() { - return [this, maybe_current_iter = std::optional{}]( - AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the call of begin() function - // If multiple begins are called before an iteration, - // one iteration will make the rest of iterators be in undefined - // states. - if (!maybe_current_iter) { - maybe_current_iter.emplace(vertices_iterable_.begin()); - } - - auto ¤t_iter{*maybe_current_iter}; - - size_t local_counter = 0; - while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) { - std::ostringstream os; - DumpVertex(&os, dba_, *current_iter); - stream->Result({TypedValue(os.str())}); - ++local_counter; - ++current_iter; - } - if (current_iter == vertices_iterable_.end()) { - return local_counter; - } - - return std::nullopt; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() { - return [this, maybe_current_vertex_iter = std::optional{}, - // we need to save the iterable which contains list of accessor so - // our saved iterator is valid in the next run - maybe_edge_iterable = std::shared_ptr{nullptr}, - maybe_current_edge_iter = std::optional{}]( - AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the call of begin() function - // If multiple begins are called before an iteration, - // one iteration will make the rest of iterators be in undefined - // states. - if (!maybe_current_vertex_iter) { - maybe_current_vertex_iter.emplace(vertices_iterable_.begin()); - } - - auto ¤t_vertex_iter{*maybe_current_vertex_iter}; - size_t local_counter = 0U; - for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) { - const auto &vertex = *current_vertex_iter; - // If we have a saved iterable from a previous pull - // we need to use the same iterable - if (!maybe_edge_iterable) { - maybe_edge_iterable = std::make_shared(vertex.OutEdges(storage::v3::View::OLD)); - } - auto &maybe_edges = *maybe_edge_iterable; - MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!"); - auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin(); - for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) { - std::ostringstream os; - DumpEdge(&os, dba_, *current_edge_iter); - stream->Result({TypedValue(os.str())}); - - ++local_counter; - } - - if (current_edge_iter != maybe_edges->end()) { - maybe_current_edge_iter.emplace(current_edge_iter); - return std::nullopt; - } - - maybe_current_edge_iter = std::nullopt; - maybe_edge_iterable = nullptr; - } - - if (current_vertex_iter == vertices_iterable_.end()) { - return local_counter; - } - - return std::nullopt; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() { - return [this](AnyStream *stream, std::optional) { - if (internal_index_created_) { - std::ostringstream os; - os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");"; - stream->Result({TypedValue(os.str())}); - return 1; - } - return 0; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() { - return [this](AnyStream *stream, std::optional) { - if (internal_index_created_) { - std::ostringstream os; - os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";"; - stream->Result({TypedValue(os.str())}); - return 1; - } - return 0; - }; -} - -void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); } - -} // namespace memgraph::query::v2 diff --git a/src/query/v2/dump.hpp b/src/query/v2/dump.hpp deleted file mode 100644 index 3c8130d8d..000000000 --- a/src/query/v2/dump.hpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include - -#include "query/v2/db_accessor.hpp" -#include "query/v2/stream.hpp" -#include "storage/v3/storage.hpp" - -namespace memgraph::query::v2 { - -void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream); - -struct PullPlanDump { - explicit PullPlanDump(query::v2::DbAccessor *dba); - - /// Pull the dump results lazily - /// @return true if all results were returned, false otherwise - bool Pull(AnyStream *stream, std::optional n); - - private: - query::v2::DbAccessor *dba_ = nullptr; - - std::optional indices_info_ = std::nullopt; - - using VertexAccessorIterable = decltype(std::declval().Vertices(storage::v3::View::OLD)); - using VertexAccessorIterableIterator = decltype(std::declval().begin()); - - using EdgeAccessorIterable = decltype(std::declval().OutEdges(storage::v3::View::OLD)); - using EdgeAccessorIterableIterator = decltype(std::declval().GetValue().begin()); - - VertexAccessorIterable vertices_iterable_; - bool internal_index_created_ = false; - - size_t current_chunk_index_ = 0; - - using PullChunk = std::function(AnyStream *stream, std::optional n)>; - // We define every part of the dump query in a self contained function. - // Each functions is responsible of keeping track of its execution status. - // If a function did finish its execution, it should return number of results - // it streamed so we know how many rows should be pulled from the next - // function, otherwise std::nullopt is returned. - std::vector pull_chunks_; - - PullChunk CreateLabelIndicesPullChunk(); - PullChunk CreateLabelPropertyIndicesPullChunk(); - PullChunk CreateInternalIndexPullChunk(); - PullChunk CreateVertexPullChunk(); - PullChunk CreateEdgePullChunk(); - PullChunk CreateDropInternalIndexPullChunk(); - PullChunk CreateInternalIndexCleanupPullChunk(); -}; -} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/ast/ast.lcp b/src/query/v2/frontend/ast/ast.lcp index 4d9ed2d8c..e30e623ea 100644 --- a/src/query/v2/frontend/ast/ast.lcp +++ b/src/query/v2/frontend/ast/ast.lcp @@ -18,6 +18,7 @@ #include #include "query/v2/bindings/ast_visitor.hpp" +#include "common/types.hpp" #include "query/v2/bindings/symbol.hpp" #include "query/v2/interpret/awesome_memgraph_functions.hpp" #include "query/v2/bindings/typed_value.hpp" diff --git a/src/query/v2/interpret/awesome_memgraph_functions.cpp b/src/query/v2/interpret/awesome_memgraph_functions.cpp index 323703bf5..46b0fb9fe 100644 --- a/src/query/v2/interpret/awesome_memgraph_functions.cpp +++ b/src/query/v2/interpret/awesome_memgraph_functions.cpp @@ -24,9 +24,6 @@ #include "query/v2/conversions.hpp" #include "query/v2/db_accessor.hpp" #include "query/v2/exceptions.hpp" -#include "query/v2/procedure/cypher_types.hpp" -//#include "query/v2/procedure/mg_procedure_impl.hpp" -//#include "query/v2/procedure/module.hpp" #include "storage/v3/conversions.hpp" #include "utils/string.hpp" #include "utils/temporal.hpp" @@ -415,48 +412,6 @@ TypedValue StartNode(const TypedValue *args, int64_t nargs, const FunctionContex return TypedValue(args[0].ValueEdge().From(), ctx.memory); } -namespace { - -size_t UnwrapDegreeResult(storage::v3::Result maybe_degree) { - if (maybe_degree.HasError()) { - switch (maybe_degree.GetError()) { - case storage::v3::Error::DELETED_OBJECT: - throw QueryRuntimeException("Trying to get degree of a deleted node."); - case storage::v3::Error::NONEXISTENT_OBJECT: - throw query::v2::QueryRuntimeException("Trying to get degree of a node that doesn't exist."); - case storage::v3::Error::SERIALIZATION_ERROR: - case storage::v3::Error::VERTEX_HAS_EDGES: - case storage::v3::Error::PROPERTIES_DISABLED: - throw QueryRuntimeException("Unexpected error when getting node degree."); - } - } - return *maybe_degree; -} - -} // namespace - -TypedValue Degree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("degree", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - const auto &vertex = args[0].ValueVertex(); - // TODO(kostasrim) Fix dummy values - return TypedValue(int64_t(0), ctx.memory); -} - -TypedValue InDegree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("inDegree", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - const auto &vertex = args[0].ValueVertex(); - return TypedValue(int64_t(0), ctx.memory); -} - -TypedValue OutDegree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("outDegree", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - const auto &vertex = args[0].ValueVertex(); - return TypedValue(int64_t(0), ctx.memory); -} - TypedValue ToBoolean(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { FType>("toBoolean", args, nargs); const auto &value = args[0]; @@ -518,9 +473,8 @@ TypedValue ToInteger(const TypedValue *args, int64_t nargs, const FunctionContex TypedValue Type(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { FType>("type", args, nargs); - auto *dba = ctx.db_accessor; if (args[0].IsNull()) return TypedValue(ctx.memory); - return TypedValue(static_cast(args[0].ValueEdge().EdgeType()), ctx.memory); + return TypedValue(args[0].ValueEdge().EdgeType().AsInt(), ctx.memory); } TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { @@ -559,30 +513,6 @@ TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContex } } -// TODO: How is Keys different from Properties function? -TypedValue Keys(const TypedValue *args, int64_t nargs, const FunctionContext & /*ctx*/) { - FType>("keys", args, nargs); - return {}; -} - -TypedValue Labels(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("labels", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - return {}; -} - -TypedValue Nodes(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("nodes", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - return {}; -} - -TypedValue Relationships(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { - FType>("relationships", args, nargs); - if (args[0].IsNull()) return TypedValue(ctx.memory); - return {}; -} - TypedValue Range(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { FType, Or, Optional>>("range", args, nargs); for (int64_t i = 0; i < nargs; ++i) @@ -1098,9 +1028,6 @@ TypedValue Duration(const TypedValue *args, int64_t nargs, const FunctionContext std::function NameToFunction( const std::string &function_name) { // Scalar functions - if (function_name == "DEGREE") return Degree; - if (function_name == "INDEGREE") return InDegree; - if (function_name == "OUTDEGREE") return OutDegree; if (function_name == "ENDNODE") return EndNode; if (function_name == "HEAD") return Head; if (function_name == kId) return Id; @@ -1116,11 +1043,7 @@ std::function #include #include +#include #include +#include "coordinator/coordinator_client.hpp" #include "expr/ast/ast_visitor.hpp" +#include "io/local_transport/local_system.hpp" +#include "io/local_transport/local_transport.hpp" #include "memory/memory_control.hpp" #include "parser/opencypher/parser.hpp" #include "query/v2/bindings/eval.hpp" @@ -33,7 +37,6 @@ #include "query/v2/context.hpp" #include "query/v2/cypher_query_interpreter.hpp" #include "query/v2/db_accessor.hpp" -#include "query/v2/dump.hpp" #include "query/v2/exceptions.hpp" #include "query/v2/frontend/ast/ast.hpp" #include "query/v2/frontend/semantic/required_privileges.hpp" @@ -41,6 +44,7 @@ #include "query/v2/plan/planner.hpp" #include "query/v2/plan/profile.hpp" #include "query/v2/plan/vertex_count_cache.hpp" +#include "query/v2/shard_request_manager.hpp" #include "storage/v3/property_value.hpp" #include "storage/v3/shard.hpp" #include "storage/v3/storage.hpp" @@ -114,17 +118,6 @@ std::optional GetOptionalValue(query::v2::Expression *expression, Expre return {}; }; -std::optional GetOptionalStringValue(query::v2::Expression *expression, ExpressionEvaluator &evaluator) { - if (expression != nullptr) { - auto value = expression->Accept(evaluator); - MG_ASSERT(value.IsNull() || value.IsString()); - if (value.IsString()) { - return {std::string(value.ValueString().begin(), value.ValueString().end())}; - } - } - return {}; -}; - class ReplQueryHandler final : public query::v2::ReplicationQueryHandler { public: explicit ReplQueryHandler(storage::v3::Shard * /*db*/) {} @@ -455,21 +448,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } } -std::optional StringPointerToOptional(const std::string *str) { - return str == nullptr ? std::nullopt : std::make_optional(*str); -} - -std::vector EvaluateTopicNames(ExpressionEvaluator &evaluator, - std::variant> topic_variant) { - return std::visit(utils::Overloaded{[&](Expression *expression) { - auto topic_names = expression->Accept(evaluator); - MG_ASSERT(topic_names.IsString()); - return utils::Split(topic_names.ValueString(), ","); - }, - [&](std::vector topic_names) { return topic_names; }}, - std::move(topic_variant)); -} - Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶meters, DbAccessor *db_accessor) { expr::Frame frame(0); SymbolTable symbol_table; @@ -671,6 +649,7 @@ struct PullPlanVector { struct PullPlan { explicit PullPlan(std::shared_ptr plan, const Parameters ¶meters, bool is_profile_query, DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory, + msgs::ShardRequestManagerInterface *shard_request_manager = nullptr, // TriggerContextCollector *trigger_context_collector = nullptr, std::optional memory_limit = {}); std::optional Pull(AnyStream *stream, std::optional n, @@ -700,7 +679,7 @@ struct PullPlan { PullPlan::PullPlan(const std::shared_ptr plan, const Parameters ¶meters, const bool is_profile_query, DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory, - const std::optional memory_limit) + msgs::ShardRequestManagerInterface *shard_request_manager, const std::optional memory_limit) // TriggerContextCollector *trigger_context_collector, const std::optional memory_limit) : plan_(plan), cursor_(plan->plan().MakeCursor(execution_memory)), @@ -710,14 +689,15 @@ PullPlan::PullPlan(const std::shared_ptr plan, const Parameters &par ctx_.symbol_table = plan->symbol_table(); ctx_.evaluation_context.timestamp = QueryTimestamp(); ctx_.evaluation_context.parameters = parameters; - ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, dba); - ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba); + ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, shard_request_manager); + ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, shard_request_manager); if (interpreter_context->config.execution_timeout_sec > 0) { ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec}; } ctx_.is_shutting_down = &interpreter_context->is_shutting_down; ctx_.is_profile_query = is_profile_query; // ctx_.trigger_context_collector = trigger_context_collector; + ctx_.shard_request_manager = shard_request_manager; } std::optional PullPlan::Pull(AnyStream *stream, std::optional n, @@ -813,13 +793,18 @@ using RWType = plan::ReadWriteTypeChecker::RWType; } // namespace InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config, - const std::filesystem::path &data_directory) - // : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory / - // "streams"} {} - : db(db), config(config) {} + const std::filesystem::path & /*data_directory*/, + io::Io io, + coordinator::Address coordinator_addr) + : db(db), config(config), io{std::move(io)}, coordinator_address{coordinator_addr} {} Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) { MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL"); + auto query_io = interpreter_context_->io.ForkLocal(); + shard_request_manager_ = std::make_unique>( + coordinator::CoordinatorClient( + query_io, interpreter_context_->coordinator_address, std::vector{interpreter_context_->coordinator_address}), + std::move(query_io)); } PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) { @@ -832,14 +817,6 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) } in_explicit_transaction_ = true; expect_rollback_ = false; - - db_accessor_ = std::make_unique( - interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride())); - execution_db_accessor_.emplace(db_accessor_.get()); - - // if (interpreter_context_->trigger_store.HasTriggers()) { - // trigger_context_collector_.emplace(interpreter_context_->trigger_store.GetEventTypes()); - // } }; } else if (query_upper == "COMMIT") { handler = [this] { @@ -886,7 +863,8 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map *summary, InterpreterContext *interpreter_context, DbAccessor *dba, - utils::MemoryResource *execution_memory, std::vector *notifications) { + utils::MemoryResource *execution_memory, std::vector *notifications, + msgs::ShardRequestManagerInterface *shard_request_manager) { // TriggerContextCollector *trigger_context_collector = nullptr) { auto *cypher_query = utils::Downcast(parsed_query.query); @@ -910,10 +888,10 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::mapplan_cache : nullptr, dba); + shard_request_manager->StartTransaction(); + auto plan = CypherQueryToPlan( + parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, parsed_query.parameters, + parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, shard_request_manager); summary->insert_or_assign("cost_estimate", plan->cost()); auto rw_type_checker = plan::ReadWriteTypeChecker(); @@ -932,7 +910,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map(plan, parsed_query.parameters, false, dba, interpreter_context, - execution_memory, memory_limit); + execution_memory, shard_request_manager, memory_limit); // execution_memory, trigger_context_collector, memory_limit); return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), [pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary]( @@ -946,7 +924,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map *summary, - InterpreterContext *interpreter_context, DbAccessor *dba, + InterpreterContext *interpreter_context, + msgs::ShardRequestManagerInterface *shard_request_manager, utils::MemoryResource *execution_memory) { const std::string kExplainQueryStart = "explain "; MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart), @@ -965,19 +944,20 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map(parsed_inner_query.query); MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN"); - auto cypher_query_plan = CypherQueryToPlan( - parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, - parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba); + auto cypher_query_plan = + CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), + cypher_query, parsed_inner_query.parameters, + parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, nullptr); std::stringstream printed_plan; - plan::PrettyPrint(*dba, &cypher_query_plan->plan(), &printed_plan); + plan::PrettyPrint(*shard_request_manager, &cypher_query_plan->plan(), &printed_plan); std::vector> printed_plan_rows; for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) { printed_plan_rows.push_back(std::vector{TypedValue(row)}); } - summary->insert_or_assign("explain", plan::PlanToJson(*dba, &cypher_query_plan->plan()).dump()); + summary->insert_or_assign("explain", plan::PlanToJson(*shard_request_manager, &cypher_query_plan->plan()).dump()); return PreparedQuery{{"QUERY PLAN"}, std::move(parsed_query.required_privileges), @@ -993,7 +973,8 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map *summary, InterpreterContext *interpreter_context, - DbAccessor *dba, utils::MemoryResource *execution_memory) { + DbAccessor *dba, utils::MemoryResource *execution_memory, + msgs::ShardRequestManagerInterface *shard_request_manager = nullptr) { const std::string kProfileQueryStart = "profile "; MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart), @@ -1042,14 +1023,15 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra auto cypher_query_plan = CypherQueryToPlan( parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, - parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba); + parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, + shard_request_manager); auto rw_type_checker = plan::ReadWriteTypeChecker(); rw_type_checker.InferRWType(const_cast(cypher_query_plan->plan())); return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"}, std::move(parsed_query.required_privileges), [plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), - summary, dba, interpreter_context, execution_memory, memory_limit, + summary, dba, interpreter_context, execution_memory, memory_limit, shard_request_manager, // We want to execute the query we are profiling lazily, so we delay // the construction of the corresponding context. stats_and_total_time = std::optional{}, @@ -1058,7 +1040,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra // No output symbols are given so that nothing is streamed. if (!stats_and_total_time) { stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context, - execution_memory, memory_limit) + execution_memory, shard_request_manager, memory_limit) .Pull(stream, {}, {}, summary); pull_plan = std::make_shared(ProfilingStatsToTable(*stats_and_total_time)); } @@ -1077,16 +1059,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map *summary, DbAccessor *dba, utils::MemoryResource *execution_memory) { - return PreparedQuery{{"QUERY"}, - std::move(parsed_query.required_privileges), - [pull_plan = std::make_shared(dba)]( - AnyStream *stream, std::optional n) -> std::optional { - if (pull_plan->Pull(stream, n)) { - return QueryHandlerResult::COMMIT; - } - return std::nullopt; - }, - RWType::R}; + throw QueryRuntimeException("Dump query is not supported!"); } PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction, @@ -1526,30 +1499,20 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query); query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count(); - // Some queries require an active transaction in order to be prepared. - if (!in_explicit_transaction_ && - (utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || - utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || - utils::Downcast(parsed_query.query))) { - db_accessor_ = std::make_unique( - interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride())); - execution_db_accessor_.emplace(db_accessor_.get()); - } - utils::Timer planning_timer; PreparedQuery prepared_query; if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, &*execution_db_accessor_, &query_execution->execution_memory, - &query_execution->notifications); + &query_execution->notifications, shard_request_manager_.get()); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, - &*execution_db_accessor_, &query_execution->execution_memory_with_exception); + &*shard_request_manager_, &query_execution->execution_memory_with_exception); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, - interpreter_context_, &*execution_db_accessor_, - &query_execution->execution_memory_with_exception); + prepared_query = PrepareProfileQuery( + std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_, + &*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get()); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_, &query_execution->execution_memory); @@ -1628,6 +1591,7 @@ void Interpreter::Commit() { // For now, we will not check if there are some unfinished queries. // We should document clearly that all results should be pulled to complete // a query. + shard_request_manager_->Commit(); if (!db_accessor_) return; const auto reset_necessary_members = [this]() { diff --git a/src/query/v2/interpreter.hpp b/src/query/v2/interpreter.hpp index dcc9bf2a7..e9925297c 100644 --- a/src/query/v2/interpreter.hpp +++ b/src/query/v2/interpreter.hpp @@ -13,6 +13,10 @@ #include +#include "coordinator/coordinator.hpp" +#include "coordinator/coordinator_client.hpp" +#include "io/local_transport/local_transport.hpp" +#include "io/transport.hpp" #include "query/v2/auth_checker.hpp" #include "query/v2/bindings/cypher_main_visitor.hpp" #include "query/v2/bindings/typed_value.hpp" @@ -165,7 +169,8 @@ struct PreparedQuery { */ struct InterpreterContext { explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config, - const std::filesystem::path &data_directory); + const std::filesystem::path &data_directory, + io::Io io, coordinator::Address coordinator_addr); storage::v3::Shard *db; @@ -180,6 +185,11 @@ struct InterpreterContext { const InterpreterConfig config; + // TODO (antaljanosbenjamin) Figure out an abstraction for io::Io to make it possible to construct an interpreter + // context with a simulator transport without templatizing it. + io::Io io; + coordinator::Address coordinator_address; + storage::v3::LabelId NameToLabelId(std::string_view label_name) { return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name)); } @@ -327,6 +337,7 @@ class Interpreter final { // move this unique_ptr into a shrared_ptr. std::unique_ptr db_accessor_; std::optional execution_db_accessor_; + std::unique_ptr shard_request_manager_; bool in_explicit_transaction_{false}; bool expect_rollback_{false}; diff --git a/src/query/v2/path.hpp b/src/query/v2/path.hpp index d16e4bba8..610de909f 100644 --- a/src/query/v2/path.hpp +++ b/src/query/v2/path.hpp @@ -14,6 +14,8 @@ #include #include +#include "query/db_accessor.hpp" +#include "query/v2/accessors.hpp" #include "query/v2/db_accessor.hpp" #include "utils/logging.hpp" #include "utils/memory.hpp" @@ -28,6 +30,8 @@ namespace memgraph::query::v2 { */ class Path { public: + using VertexAccessor = accessors::VertexAccessor; + using EdgeAccessor = accessors::EdgeAccessor; /** Allocator type so that STL containers are aware that we need one */ using allocator_type = utils::Allocator; diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 320c2bfd0..0a67169f0 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -27,6 +27,7 @@ #include #include "expr/exceptions.hpp" +#include "query/exceptions.hpp" #include "query/v2/accessors.hpp" #include "query/v2/bindings/eval.hpp" #include "query/v2/bindings/symbol_table.hpp" @@ -36,7 +37,6 @@ #include "query/v2/frontend/ast/ast.hpp" #include "query/v2/path.hpp" #include "query/v2/plan/scoped_profile.hpp" -#include "query/v2/procedure/cypher_types.hpp" #include "query/v2/requests.hpp" #include "query/v2/shard_request_manager.hpp" #include "storage/v3/conversions.hpp" @@ -156,6 +156,71 @@ uint64_t ComputeProfilingKey(const T *obj) { #define SCOPED_PROFILE_OP(name) ScopedProfile profile{ComputeProfilingKey(this), name, &context}; +class DistributedCreateNodeCursor : public Cursor { + public: + using InputOperator = std::shared_ptr; + DistributedCreateNodeCursor(const InputOperator &op, utils::MemoryResource *mem, + std::vector nodes_info) + : input_cursor_(op->MakeCursor(mem)), nodes_info_(std::move(nodes_info)) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("CreateNode"); + if (input_cursor_->Pull(frame, context)) { + auto &shard_manager = context.shard_request_manager; + shard_manager->Request(state_, NodeCreationInfoToRequest(context, frame)); + return true; + } + + return false; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { state_ = {}; } + + std::vector NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) const { + std::vector requests; + for (const auto &node_info : nodes_info_) { + msgs::NewVertex rqst; + std::map properties; + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, + storage::v3::View::NEW); + if (const auto *node_info_properties = std::get_if(&node_info->properties)) { + for (const auto &[key, value_expression] : *node_info_properties) { + TypedValue val = value_expression->Accept(evaluator); + properties[key] = TypedValueToValue(val); + if (context.shard_request_manager->IsPrimaryKey(key)) { + rqst.primary_key.push_back(storage::v3::TypedValueToValue(val)); + } + } + } else { + auto property_map = evaluator.Visit(*std::get(node_info->properties)).ValueMap(); + for (const auto &[key, value] : property_map) { + auto key_str = std::string(key); + auto property_id = context.shard_request_manager->NameToProperty(key_str); + properties[property_id] = TypedValueToValue(value); + if (context.shard_request_manager->IsPrimaryKey(property_id)) { + rqst.primary_key.push_back(storage::v3::TypedValueToValue(value)); + } + } + } + + if (node_info->labels.empty()) { + throw QueryRuntimeException("Primary label must be defined!"); + } + // TODO(kostasrim) Copy non primary labels as well + rqst.label_ids.push_back(msgs::Label{node_info->labels[0]}); + requests.push_back(std::move(rqst)); + } + return requests; + } + + private: + const UniqueCursorPtr input_cursor_; + std::vector nodes_info_; + msgs::ExecutionState state_; +}; + bool Once::OnceCursor::Pull(Frame &, ExecutionContext &context) { SCOPED_PROFILE_OP("Once"); @@ -186,7 +251,7 @@ ACCEPT_WITH_INPUT(CreateNode) UniqueCursorPtr CreateNode::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::CreateNodeOperator); - return MakeUniqueCursorPtr(mem, *this, mem); + return MakeUniqueCursorPtr(mem, input_, mem, std::vector{&this->node_info_}); } std::vector CreateNode::ModifiedSymbols(const SymbolTable &table) const { @@ -272,15 +337,12 @@ ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_sy ACCEPT_WITH_INPUT(ScanAll) +class DistributedScanAllCursor; + UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::ScanAllOperator); - auto vertices = [this](Frame & /*unused*/, ExecutionContext &context) { - auto *db = context.db_accessor; - return std::make_optional(db->Vertices(view_)); - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAll"); + return MakeUniqueCursorPtr(mem, output_symbol_, input_->MakeCursor(mem), "ScanAll"); } std::vector ScanAll::ModifiedSymbols(const SymbolTable &table) const { @@ -295,15 +357,10 @@ ScanAllByLabel::ScanAllByLabel(const std::shared_ptr &input, Sy ACCEPT_WITH_INPUT(ScanAllByLabel) -UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const { +UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource * /*mem*/) const { EventCounter::IncrementCounter(EventCounter::ScanAllByLabelOperator); - auto vertices = [this](Frame & /*unused*/, ExecutionContext &context) { - auto *db = context.db_accessor; - return std::make_optional(db->Vertices(view_, label_)); - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAllByLabel"); + throw QueryRuntimeException("ScanAllByLabel is not supported"); } // TODO(buda): Implement ScanAllByLabelProperty operator to iterate over @@ -326,50 +383,10 @@ ScanAllByLabelPropertyRange::ScanAllByLabelPropertyRange(const std::shared_ptr std::optionalVertices(view_, label_, property_, std::nullopt, std::nullopt))> { - auto *db = context.db_accessor; - ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_); - auto convert = [&evaluator](const auto &bound) -> std::optional> { - if (!bound) return std::nullopt; - const auto &value = bound->value()->Accept(evaluator); - try { - const auto &property_value = storage::v3::TypedToPropertyValue(value); - switch (property_value.type()) { - case storage::v3::PropertyValue::Type::Bool: - case storage::v3::PropertyValue::Type::List: - case storage::v3::PropertyValue::Type::Map: - // Prevent indexed lookup with something that would fail if we did - // the original filter with `operator<`. Note, for some reason, - // Cypher does not support comparing boolean values. - throw QueryRuntimeException("Invalid type {} for '<'.", value.type()); - case storage::v3::PropertyValue::Type::Null: - case storage::v3::PropertyValue::Type::Int: - case storage::v3::PropertyValue::Type::Double: - case storage::v3::PropertyValue::Type::String: - case storage::v3::PropertyValue::Type::TemporalData: - // These are all fine, there's also Point, Date and Time data types - // which were added to Cypher, but we don't have support for those - // yet. - return std::make_optional(utils::Bound(property_value, bound->type())); - } - } catch (const expr::TypedValueException &) { - throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type()); - } - }; - auto maybe_lower = convert(lower_bound_); - auto maybe_upper = convert(upper_bound_); - // If any bound is null, then the comparison would result in nulls. This - // is treated as not satisfying the filter, so return no vertices. - if (maybe_lower && maybe_lower->value().IsNull()) return std::nullopt; - if (maybe_upper && maybe_upper->value().IsNull()) return std::nullopt; - return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper)); - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAllByLabelPropertyRange"); + throw QueryRuntimeException("ScanAllByLabelPropertyRange is not supported"); } ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr &input, @@ -387,20 +404,10 @@ ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr std::optionalVertices( - view_, label_, property_, storage::v3::PropertyValue()))> { - auto *db = context.db_accessor; - ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_); - auto value = expression_->Accept(evaluator); - if (value.IsNull()) return std::nullopt; - return std::make_optional(db->Vertices(view_, label_, property_, storage::v3::TypedToPropertyValue(value))); - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAllByLabelPropertyValue"); + throw QueryRuntimeException("ScanAllByLabelPropertyValue is not supported"); } ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, @@ -412,13 +419,7 @@ ACCEPT_WITH_INPUT(ScanAllByLabelProperty) UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyOperator); - - auto vertices = [this](Frame & /*frame*/, ExecutionContext &context) { - auto *db = context.db_accessor; - return std::make_optional(db->Vertices(view_, label_, property_)); - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAllByLabelProperty"); + throw QueryRuntimeException("ScanAllByLabelProperty is not supported"); } ScanAllById::ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, @@ -431,16 +432,8 @@ ACCEPT_WITH_INPUT(ScanAllById) UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::ScanAllByIdOperator); - // TODO Reimplement when we have reliable conversion between hash value and pk - auto vertices = [this](Frame &frame, ExecutionContext &context) -> std::optional> { - // auto *db = context.db_accessor; - // ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, - // view_); auto value = expression_->Accept(evaluator); if (!value.IsNumeric()) return std::nullopt; int64_t id = - // value.IsInt() ? value.ValueInt() : value.ValueDouble(); if (value.IsDouble() && id != value.ValueDouble()) return - // std::nullopt; auto maybe_vertex = db->FindVertex(storage::v3::Gid::FromInt(id), view_); auto maybe_vertex = - // nullptr; if (!maybe_vertex) return std::nullopt; + auto vertices = [](Frame & /*frame*/, ExecutionContext & /*context*/) -> std::optional> { return std::nullopt; - // return std::vector{*maybe_vertex}; }; return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), std::move(vertices), "ScanAllById"); @@ -540,265 +533,10 @@ std::vector ExpandVariable::ModifiedSymbols(const SymbolTable &table) co return symbols; } -namespace { - -/** - * Helper function that returns an iterable over - * pairs - * for the given params. - * - * @param vertex - The vertex to expand from. - * @param direction - Expansion direction. All directions (IN, OUT, BOTH) - * are supported. - * @param memory - Used to allocate the result. - * @return See above. - */ -auto ExpandFromVertex(const VertexAccessor &vertex, EdgeAtom::Direction direction, - const std::vector &edge_types, utils::MemoryResource *memory) { - // wraps an EdgeAccessor into a pair - auto wrapper = [](EdgeAtom::Direction direction, auto &&edges) { - return iter::imap([direction](const auto &edge) { return std::make_pair(edge, direction); }, - std::forward(edges)); - }; - - storage::v3::View view = storage::v3::View::OLD; - utils::pmr::vector chain_elements(memory); - - if (direction != EdgeAtom::Direction::OUT) { - auto edges = UnwrapEdgesResult(vertex.InEdges(view, edge_types)); - if (edges.begin() != edges.end()) { - chain_elements.emplace_back(wrapper(EdgeAtom::Direction::IN, std::move(edges))); - } - } - if (direction != EdgeAtom::Direction::IN) { - auto edges = UnwrapEdgesResult(vertex.OutEdges(view, edge_types)); - if (edges.begin() != edges.end()) { - chain_elements.emplace_back(wrapper(EdgeAtom::Direction::OUT, std::move(edges))); - } - } - - // TODO: Investigate whether itertools perform heap allocation? - return iter::chain.from_iterable(std::move(chain_elements)); -} - -} // namespace - -class ExpandVariableCursor : public Cursor { - public: - ExpandVariableCursor(const ExpandVariable &self, utils::MemoryResource *mem) - : self_(self), input_cursor_(self.input_->MakeCursor(mem)), edges_(mem), edges_it_(mem) {} - - bool Pull(Frame & /*frame*/, ExecutionContext & /*context*/) override { return false; } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { - input_cursor_->Reset(); - edges_.clear(); - edges_it_.clear(); - } - - private: - const ExpandVariable &self_; - const UniqueCursorPtr input_cursor_; - // bounds. in the cursor they are not optional but set to - // default values if missing in the ExpandVariable operator - // initialize to arbitrary values, they should only be used - // after a successful pull from the input - int64_t upper_bound_{-1}; - int64_t lower_bound_{-1}; - - // a stack of edge iterables corresponding to the level/depth of - // the expansion currently being Pulled - using ExpandEdges = decltype(ExpandFromVertex(std::declval(), EdgeAtom::Direction::IN, - self_.common_.edge_types, utils::NewDeleteResource())); - - utils::pmr::vector edges_; - // an iterator indicating the position in the corresponding edges_ element - utils::pmr::vectorbegin())> edges_it_; - - /** - * Performs a single expansion for the current state of this - * VariableExpansionCursor. - * - * @return True if the expansion was a success and this Cursor's - * consumer can consume it. False if the expansion failed. In that - * case no more expansions are available from the current input - * vertex and another Pull from the input cursor should be performed. - */ - // NOLINTNEXTLINE(readability-convert-member-functions-to-static) - bool Expand(Frame & /*frame*/, ExecutionContext & /*context*/) { return false; } -}; - -class STShortestPathCursor : public query::v2::plan::Cursor { - public: - STShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) - : self_(self), input_cursor_(self_.input()->MakeCursor(mem)) { - MG_ASSERT(self_.common_.existing_node, - "s-t shortest path algorithm should only " - "be used when `existing_node` flag is " - "set!"); - } - - bool Pull(Frame & /*frame*/, ExecutionContext & /*context*/) override { return false; } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { input_cursor_->Reset(); } - - private: - const ExpandVariable &self_; - UniqueCursorPtr input_cursor_; - - using VertexEdgeMapT = utils::pmr::unordered_map>; -}; - -class SingleSourceShortestPathCursor : public query::v2::plan::Cursor { - public: - SingleSourceShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) - : self_(self), - input_cursor_(self_.input()->MakeCursor(mem)), - processed_(mem), - to_visit_current_(mem), - to_visit_next_(mem) { - MG_ASSERT(!self_.common_.existing_node, - "Single source shortest path algorithm " - "should not be used when `existing_node` " - "flag is set, s-t shortest path algorithm " - "should be used instead!"); - } - - bool Pull(Frame & /*frame*/, ExecutionContext & /*context*/) override { return true; } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { - input_cursor_->Reset(); - processed_.clear(); - to_visit_next_.clear(); - to_visit_current_.clear(); - } - - private: - const ExpandVariable &self_; - const UniqueCursorPtr input_cursor_; - - // Depth bounds. Calculated on each pull from the input, the initial value - // is irrelevant. - int64_t lower_bound_{-1}; - int64_t upper_bound_{-1}; - - // maps vertices to the edge they got expanded from. it is an optional - // edge because the root does not get expanded from anything. - // contains visited vertices as well as those scheduled to be visited. - utils::pmr::unordered_map> processed_; - // edge/vertex pairs we have yet to visit, for current and next depth - utils::pmr::vector> to_visit_current_; - utils::pmr::vector> to_visit_next_; -}; - -class ExpandWeightedShortestPathCursor : public query::v2::plan::Cursor { - public: - ExpandWeightedShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) - : self_(self), - input_cursor_(self_.input_->MakeCursor(mem)), - total_cost_(mem), - previous_(mem), - yielded_vertices_(mem), - pq_(mem) {} - - bool Pull(Frame & /*frame*/, ExecutionContext & /*context*/) override { return false; } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { - input_cursor_->Reset(); - previous_.clear(); - total_cost_.clear(); - yielded_vertices_.clear(); - ClearQueue(); - } - - private: - const ExpandVariable &self_; - const UniqueCursorPtr input_cursor_; - - // Upper bound on the path length. - int64_t upper_bound_{-1}; - bool upper_bound_set_{false}; - - struct WspStateHash { - size_t operator()(const std::pair &key) const { - return utils::HashCombine{}(key.first, key.second); - } - }; - - // Maps vertices to weights they got in expansion. - utils::pmr::unordered_map, TypedValue, WspStateHash> total_cost_; - - // Maps vertices to edges used to reach them. - utils::pmr::unordered_map, std::optional, WspStateHash> previous_; - - // Keeps track of vertices for which we yielded a path already. - utils::pmr::unordered_set yielded_vertices_; - - static void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) { - if (!((lhs.IsNumeric() && lhs.IsNumeric()) || (rhs.IsDuration() && rhs.IsDuration()))) { - throw QueryRuntimeException(utils::MessageWithLink( - "All weights should be of the same type, either numeric or a Duration. Please update the weight " - "expression or the filter expression.", - "https://memgr.ph/wsp")); - } - } - - // Priority queue comparator. Keep lowest weight on top of the queue. - class PriorityQueueComparator { - public: - bool operator()(const std::tuple> &lhs, - const std::tuple> &rhs) { - const auto &lhs_weight = std::get<0>(lhs); - const auto &rhs_weight = std::get<0>(rhs); - // Null defines minimum value for all types - if (lhs_weight.IsNull()) { - return false; - } - - if (rhs_weight.IsNull()) { - return true; - } - - ValidateWeightTypes(lhs_weight, rhs_weight); - return (lhs_weight > rhs_weight).ValueBool(); - } - }; - - std::priority_queue>, - utils::pmr::vector>>, - PriorityQueueComparator> - pq_; - - void ClearQueue() { - while (!pq_.empty()) pq_.pop(); - } -}; - UniqueCursorPtr ExpandVariable::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::ExpandVariableOperator); - switch (type_) { - case EdgeAtom::Type::BREADTH_FIRST: - if (common_.existing_node) { - return MakeUniqueCursorPtr(mem, *this, mem); - } else { - return MakeUniqueCursorPtr(mem, *this, mem); - } - case EdgeAtom::Type::DEPTH_FIRST: - return MakeUniqueCursorPtr(mem, *this, mem); - case EdgeAtom::Type::WEIGHTED_SHORTEST_PATH: - return MakeUniqueCursorPtr(mem, *this, mem); - case EdgeAtom::Type::SINGLE: - LOG_FATAL("ExpandVariable should not be planned for a single expansion!"); - } + throw QueryRuntimeException("ExpandVariable is not supported"); } class ConstructNamedPathCursor : public Cursor { @@ -1039,43 +777,6 @@ SetLabels::SetLabelsCursor::SetLabelsCursor(const SetLabels &self, utils::Memory bool SetLabels::SetLabelsCursor::Pull(Frame &frame, ExecutionContext &context) { SCOPED_PROFILE_OP("SetLabels"); return false; - // if (!input_cursor_->Pull(frame, context)) return false; - // - // TypedValue &vertex_value = frame[self_.input_symbol_]; - // // Skip setting labels on Null (can occur in optional match). - // if (vertex_value.IsNull()) return true; - // ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); - // - // auto &dba = *context.db_accessor; - // auto &vertex = vertex_value.ValueVertex(); - // for (const auto label : self_.labels_) { - // auto maybe_value = vertex.AddLabelAndValidate(label); - // if (maybe_value.HasError()) { - // std::visit(utils::Overloaded{[](const storage::v3::Error error) { - // switch (error) { - // case storage::v3::Error::SERIALIZATION_ERROR: - // throw TransactionSerializationException(); - // case storage::v3::Error::DELETED_OBJECT: - // throw QueryRuntimeException("Trying to set a label on a deleted node."); - // case storage::v3::Error::VERTEX_HAS_EDGES: - // case storage::v3::Error::PROPERTIES_DISABLED: - // case storage::v3::Error::NONEXISTENT_OBJECT: - // throw QueryRuntimeException("Unexpected error when setting a label."); - // } - // }, - // [&dba](const storage::v3::SchemaViolation schema_violation) { - // HandleSchemaViolation(schema_violation, dba); - // }}, - // maybe_value.GetError()); - // } - // - // context.execution_stats[ExecutionStats::Key::CREATED_LABELS]++; - // if (context.trigger_context_collector && *maybe_value) { - // context.trigger_context_collector->RegisterSetVertexLabel(vertex, label); - // } - // } - // - // return true; } void SetLabels::SetLabelsCursor::Shutdown() { input_cursor_->Shutdown(); } @@ -1104,36 +805,6 @@ RemoveProperty::RemovePropertyCursor::RemovePropertyCursor(const RemoveProperty bool RemoveProperty::RemovePropertyCursor::Pull(Frame &frame, ExecutionContext &context) { SCOPED_PROFILE_OP("RemoveProperty"); return false; - // if (!input_cursor_->Pull(frame, context)) return false; - // - // // Remove, just like Delete needs to see the latest changes. - // ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, - // storage::v3::View::NEW); - // TypedValue lhs = self_.lhs_->expression_->Accept(evaluator); - // - // auto remove_prop = [property = self_.property_, &context](auto *record) { - // auto old_value = PropsSetChecked(record, *context.db_accessor, property, TypedValue{}); - // - // if (context.trigger_context_collector) { - // context.trigger_context_collector->RegisterRemovedObjectProperty( - // *record, property, storage::v3::PropertyToTypedValue(std::move(old_value))); - // } - // }; - // - // switch (lhs.type()) { - // case TypedValue::Type::Vertex: - // remove_prop(&lhs.ValueVertex()); - // break; - // case TypedValue::Type::Edge: - // remove_prop(&lhs.ValueEdge()); - // break; - // case TypedValue::Type::Null: - // // Skip removing properties on Null (can occur in optional match). - // break; - // default: - // throw QueryRuntimeException("Properties can only be removed from vertices and edges."); - // } - // return true; } void RemoveProperty::RemovePropertyCursor::Shutdown() { input_cursor_->Shutdown(); } @@ -1162,44 +833,6 @@ RemoveLabels::RemoveLabelsCursor::RemoveLabelsCursor(const RemoveLabels &self, u bool RemoveLabels::RemoveLabelsCursor::Pull(Frame &frame, ExecutionContext &context) { SCOPED_PROFILE_OP("RemoveLabels"); return false; - // - // if (!input_cursor_->Pull(frame, context)) return false; - // - // TypedValue &vertex_value = frame[self_.input_symbol_]; - // // Skip removing labels on Null (can occur in optional match). - // if (vertex_value.IsNull()) return true; - // ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); - // auto &vertex = vertex_value.ValueVertex(); - // for (auto label : self_.labels_) { - // auto maybe_value = vertex.RemoveLabelAndValidate(label); - // if (maybe_value.HasError()) { - // std::visit( - // utils::Overloaded{[](const storage::v3::Error error) { - // switch (error) { - // case storage::v3::Error::SERIALIZATION_ERROR: - // throw TransactionSerializationException(); - // case storage::v3::Error::DELETED_OBJECT: - // throw QueryRuntimeException("Trying to remove labels from a deleted node."); - // case storage::v3::Error::VERTEX_HAS_EDGES: - // case storage::v3::Error::PROPERTIES_DISABLED: - // case storage::v3::Error::NONEXISTENT_OBJECT: - // throw QueryRuntimeException("Unexpected error when removing labels from a - // node."); - // } - // }, - // [&context](const storage::v3::SchemaViolation &schema_violation) { - // HandleSchemaViolation(schema_violation, *context.db_accessor); - // }}, - // maybe_value.GetError()); - // } - // - // context.execution_stats[ExecutionStats::Key::DELETED_LABELS] += 1; - // if (context.trigger_context_collector && *maybe_value) { - // context.trigger_context_collector->RegisterRemovedVertexLabel(vertex, label); - // } - // } - // - // return true; } void RemoveLabels::RemoveLabelsCursor::Shutdown() { input_cursor_->Shutdown(); } @@ -1278,57 +911,9 @@ ACCEPT_WITH_INPUT(Accumulate) std::vector Accumulate::ModifiedSymbols(const SymbolTable &) const { return symbols_; } -class AccumulateCursor : public Cursor { - public: - AccumulateCursor(const Accumulate &self, utils::MemoryResource *mem) - : self_(self), input_cursor_(self.input_->MakeCursor(mem)), cache_(mem) {} - - bool Pull(Frame &frame, ExecutionContext &context) override { - SCOPED_PROFILE_OP("Accumulate"); - - auto &dba = *context.db_accessor; - // cache all the input - if (!pulled_all_input_) { - while (input_cursor_->Pull(frame, context)) { - utils::pmr::vector row(cache_.get_allocator().GetMemoryResource()); - row.reserve(self_.symbols_.size()); - for (const Symbol &symbol : self_.symbols_) row.emplace_back(frame[symbol]); - cache_.emplace_back(std::move(row)); - } - pulled_all_input_ = true; - cache_it_ = cache_.begin(); - - if (self_.advance_command_) dba.AdvanceCommand(); - } - - if (MustAbort(context)) throw HintedAbortError(); - if (cache_it_ == cache_.end()) return false; - auto row_it = (cache_it_++)->begin(); - for (const Symbol &symbol : self_.symbols_) frame[symbol] = *row_it++; - return true; - } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { - input_cursor_->Reset(); - cache_.clear(); - cache_it_ = cache_.begin(); - pulled_all_input_ = false; - } - - private: - const Accumulate &self_; - const UniqueCursorPtr input_cursor_; - utils::pmr::vector> cache_; - decltype(cache_.begin()) cache_it_ = cache_.begin(); - bool pulled_all_input_{false}; -}; - UniqueCursorPtr Accumulate::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::AccumulateOperator); - - return MakeUniqueCursorPtr(mem, *this, mem); + throw QueryRuntimeException("Accumulate is not supported"); } Aggregate::Aggregate(const std::shared_ptr &input, const std::vector &aggregations, @@ -2448,40 +2033,8 @@ std::unordered_map CallProcedure::GetAndResetCounters() { return ret; } -class CallProcedureCursor : public Cursor { - const CallProcedure *self_; - UniqueCursorPtr input_cursor_; - mgp_result result_; - decltype(result_.rows.end()) result_row_it_{result_.rows.end()}; - size_t result_signature_size_{0}; - - public: - CallProcedureCursor(const CallProcedure *self, utils::MemoryResource *mem) - : self_(self), - input_cursor_(self_->input_->MakeCursor(mem)), - // result_ needs to live throughout multiple Pull evaluations, until all - // rows are produced. Therefore, we use the memory dedicated for the - // whole execution. - result_(nullptr, mem) { - MG_ASSERT(self_->result_fields_.size() == self_->result_symbols_.size(), "Incorrectly constructed CallProcedure"); - } - - bool Pull(Frame & /*frame*/, ExecutionContext & /*context*/) override { return false; } - - void Reset() override { - result_.rows.clear(); - result_.error_msg.reset(); - input_cursor_->Reset(); - } - - void Shutdown() override {} -}; - UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const { - EventCounter::IncrementCounter(EventCounter::CallProcedureOperator); - CallProcedure::IncrementCounter(procedure_name_); - - return MakeUniqueCursorPtr(mem, this, mem); + throw QueryRuntimeException("Procedure call is not supported!"); } LoadCsv::LoadCsv(std::shared_ptr input, Expression *file, bool with_header, bool ignore_bad, @@ -2711,6 +2264,8 @@ class DistributedScanAllCursor : public Cursor { using VertexAccessor = accessors::VertexAccessor; bool MakeRequest(msgs::ShardRequestManagerInterface &shard_manager) { + // TODO(antaljanosbenjamin) Use real label + request_state_.label = "label"; current_batch = shard_manager.Request(request_state_); current_vertex_it = current_batch.begin(); return !current_batch.empty(); @@ -2759,69 +2314,4 @@ class DistributedScanAllCursor : public Cursor { decltype(std::vector().begin()) current_vertex_it; msgs::ExecutionState request_state_; }; - -class DistributedCreateNodeCursor : public Cursor { - public: - using InputOperator = std::shared_ptr; - DistributedCreateNodeCursor(const InputOperator &op, utils::MemoryResource *mem, - std::vector nodes_info) - : input_cursor_(op->MakeCursor(mem)), nodes_info_(std::move(nodes_info)) {} - - bool Pull(Frame &frame, ExecutionContext &context) override { - SCOPED_PROFILE_OP("CreateNode"); - if (input_cursor_->Pull(frame, context)) { - auto &shard_manager = context.shard_request_manager; - shard_manager->Request(state_, NodeCreationInfoToRequest(context, frame)); - return true; - } - - return false; - } - - void Shutdown() override { input_cursor_->Shutdown(); } - - void Reset() override { state_ = {}; } - - std::vector NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) const { - std::vector requests; - for (const auto &node_info : nodes_info_) { - msgs::NewVertex rqst; - std::map properties; - ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, - storage::v3::View::NEW); - if (const auto *node_info_properties = std::get_if(&node_info.properties)) { - for (const auto &[key, value_expression] : *node_info_properties) { - TypedValue val = value_expression->Accept(evaluator); - properties[key] = TypedValueToValue(val); - if (context.shard_request_manager->IsPrimaryKey(key)) { - rqst.primary_key.push_back(storage::v3::TypedValueToValue(val)); - } - } - } else { - auto property_map = evaluator.Visit(*std::get(node_info.properties)).ValueMap(); - for (const auto &[key, value] : property_map) { - auto key_str = std::string(key); - auto property_id = context.shard_request_manager->NameToProperty(key_str); - properties[property_id] = TypedValueToValue(value); - if (context.shard_request_manager->IsPrimaryKey(property_id)) { - rqst.primary_key.push_back(storage::v3::TypedValueToValue(value)); - } - } - } - - if (node_info.labels.empty()) { - throw QueryRuntimeException("Primary label must be defined!"); - } - // TODO(kostasrim) Copy non primary labels as well - rqst.label_ids.push_back(msgs::Label{node_info.labels[0]}); - requests.push_back(std::move(rqst)); - } - return requests; - } - - private: - const UniqueCursorPtr input_cursor_; - std::vector nodes_info_; - msgs::ExecutionState state_; -}; } // namespace memgraph::query::v2::plan diff --git a/src/query/v2/plan/pretty_print.cpp b/src/query/v2/plan/pretty_print.cpp index 34d815aff..b756a6299 100644 --- a/src/query/v2/plan/pretty_print.cpp +++ b/src/query/v2/plan/pretty_print.cpp @@ -14,11 +14,13 @@ #include "query/v2/bindings/pretty_print.hpp" #include "query/v2/db_accessor.hpp" +#include "query/v2/shard_request_manager.hpp" #include "utils/string.hpp" namespace memgraph::query::v2::plan { -PlanPrinter::PlanPrinter(const DbAccessor *dba, std::ostream *out) : dba_(dba), out_(out) {} +PlanPrinter::PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out) + : request_manager_(request_manager), out_(out) {} #define PRE_VISIT(TOp) \ bool PlanPrinter::PreVisit(TOp &) { \ @@ -32,7 +34,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) { WithPrintLn([&](auto &out) { out << "* CreateExpand (" << op.input_symbol_.name() << ")" << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "[" - << op.edge_info_.symbol.name() << ":" << dba_->EdgeTypeToName(op.edge_info_.edge_type) << "]" + << op.edge_info_.symbol.name() << ":" << request_manager_->EdgeTypeToName(op.edge_info_.edge_type) << "]" << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "(" << op.node_info_.symbol.name() << ")"; }); @@ -52,7 +54,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) { bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) { WithPrintLn([&](auto &out) { out << "* ScanAllByLabel" - << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << ")"; + << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")"; }); return true; } @@ -60,8 +62,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) { bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) { WithPrintLn([&](auto &out) { out << "* ScanAllByLabelPropertyValue" - << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {" - << dba_->PropertyToName(op.property_) << "})"; + << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {" + << request_manager_->PropertyToName(op.property_) << "})"; }); return true; } @@ -69,8 +71,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) { bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) { WithPrintLn([&](auto &out) { out << "* ScanAllByLabelPropertyRange" - << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {" - << dba_->PropertyToName(op.property_) << "})"; + << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {" + << request_manager_->PropertyToName(op.property_) << "})"; }); return true; } @@ -78,8 +80,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) { bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) { WithPrintLn([&](auto &out) { out << "* ScanAllByLabelProperty" - << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {" - << dba_->PropertyToName(op.property_) << "})"; + << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {" + << request_manager_->PropertyToName(op.property_) << "})"; }); return true; } @@ -98,7 +100,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) { << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "[" << op.common_.edge_symbol.name(); utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) { - stream << ":" << dba_->EdgeTypeToName(edge_type); + stream << ":" << request_manager_->EdgeTypeToName(edge_type); }); *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "(" << op.common_.node_symbol.name() << ")"; @@ -127,7 +129,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) { << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "[" << op.common_.edge_symbol.name(); utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) { - stream << ":" << dba_->EdgeTypeToName(edge_type); + stream << ":" << request_manager_->EdgeTypeToName(edge_type); }); *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "(" << op.common_.node_symbol.name() << ")"; @@ -261,14 +263,15 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string --depth_; } -void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out) { - PlanPrinter printer(&dba, out); +void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root, + std::ostream *out) { + PlanPrinter printer(&request_manager, out); // FIXME(mtomic): We should make visitors that take const arguments. const_cast(plan_root)->Accept(printer); } -nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root) { - impl::PlanToJsonVisitor visitor(&dba); +nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) { + impl::PlanToJsonVisitor visitor(&request_manager); // FIXME(mtomic): We should make visitors that take const arguments. const_cast(plan_root)->Accept(visitor); return visitor.output(); @@ -346,11 +349,17 @@ json ToJson(const utils::Bound &bound) { json ToJson(const Symbol &symbol) { return symbol.name(); } -json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba) { return dba.EdgeTypeToName(edge_type); } +json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager) { + return request_manager.EdgeTypeToName(edge_type); +} -json ToJson(storage::v3::LabelId label, const DbAccessor &dba) { return dba.LabelToName(label); } +json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager) { + return request_manager.LabelToName(label); +} -json ToJson(storage::v3::PropertyId property, const DbAccessor &dba) { return dba.PropertyToName(property); } +json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager) { + return request_manager.PropertyToName(property); +} json ToJson(NamedExpression *nexpr) { json json; @@ -359,29 +368,30 @@ json ToJson(NamedExpression *nexpr) { return json; } -json ToJson(const std::vector> &properties, const DbAccessor &dba) { +json ToJson(const std::vector> &properties, + const msgs::ShardRequestManagerInterface &request_manager) { json json; for (const auto &prop_pair : properties) { - json.emplace(ToJson(prop_pair.first, dba), ToJson(prop_pair.second)); + json.emplace(ToJson(prop_pair.first, request_manager), ToJson(prop_pair.second)); } return json; } -json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba) { +json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager) { json self; self["symbol"] = ToJson(node_info.symbol); - self["labels"] = ToJson(node_info.labels, dba); + self["labels"] = ToJson(node_info.labels, request_manager); const auto *props = std::get_if(&node_info.properties); - self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba); + self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager); return self; } -json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba) { +json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager) { json self; self["symbol"] = ToJson(edge_info.symbol); const auto *props = std::get_if(&edge_info.properties); - self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba); - self["edge_type"] = ToJson(edge_info.edge_type, dba); + self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager); + self["edge_type"] = ToJson(edge_info.edge_type, request_manager); self["direction"] = ToString(edge_info.direction); return self; } @@ -423,7 +433,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) { bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) { json self; self["name"] = "ScanAllByLabel"; - self["label"] = ToJson(op.label_, *dba_); + self["label"] = ToJson(op.label_, *request_manager_); self["output_symbol"] = ToJson(op.output_symbol_); op.input_->Accept(*this); @@ -436,8 +446,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) { bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) { json self; self["name"] = "ScanAllByLabelPropertyRange"; - self["label"] = ToJson(op.label_, *dba_); - self["property"] = ToJson(op.property_, *dba_); + self["label"] = ToJson(op.label_, *request_manager_); + self["property"] = ToJson(op.property_, *request_manager_); self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json(); self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json(); self["output_symbol"] = ToJson(op.output_symbol_); @@ -452,8 +462,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) { bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) { json self; self["name"] = "ScanAllByLabelPropertyValue"; - self["label"] = ToJson(op.label_, *dba_); - self["property"] = ToJson(op.property_, *dba_); + self["label"] = ToJson(op.label_, *request_manager_); + self["property"] = ToJson(op.property_, *request_manager_); self["expression"] = ToJson(op.expression_); self["output_symbol"] = ToJson(op.output_symbol_); @@ -467,8 +477,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) { bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) { json self; self["name"] = "ScanAllByLabelProperty"; - self["label"] = ToJson(op.label_, *dba_); - self["property"] = ToJson(op.property_, *dba_); + self["label"] = ToJson(op.label_, *request_manager_); + self["property"] = ToJson(op.property_, *request_manager_); self["output_symbol"] = ToJson(op.output_symbol_); op.input_->Accept(*this); @@ -491,7 +501,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAllById &op) { bool PlanToJsonVisitor::PreVisit(CreateNode &op) { json self; self["name"] = "CreateNode"; - self["node_info"] = ToJson(op.node_info_, *dba_); + self["node_info"] = ToJson(op.node_info_, *request_manager_); op.input_->Accept(*this); self["input"] = PopOutput(); @@ -504,8 +514,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) { json self; self["name"] = "CreateExpand"; self["input_symbol"] = ToJson(op.input_symbol_); - self["node_info"] = ToJson(op.node_info_, *dba_); - self["edge_info"] = ToJson(op.edge_info_, *dba_); + self["node_info"] = ToJson(op.node_info_, *request_manager_); + self["edge_info"] = ToJson(op.edge_info_, *request_manager_); self["existing_node"] = op.existing_node_; op.input_->Accept(*this); @@ -521,7 +531,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) { self["input_symbol"] = ToJson(op.input_symbol_); self["node_symbol"] = ToJson(op.common_.node_symbol); self["edge_symbol"] = ToJson(op.common_.edge_symbol); - self["edge_types"] = ToJson(op.common_.edge_types, *dba_); + self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_); self["direction"] = ToString(op.common_.direction); self["existing_node"] = op.common_.existing_node; @@ -538,7 +548,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) { self["input_symbol"] = ToJson(op.input_symbol_); self["node_symbol"] = ToJson(op.common_.node_symbol); self["edge_symbol"] = ToJson(op.common_.edge_symbol); - self["edge_types"] = ToJson(op.common_.edge_types, *dba_); + self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_); self["direction"] = ToString(op.common_.direction); self["type"] = ToString(op.type_); self["is_reverse"] = op.is_reverse_; @@ -613,7 +623,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) { bool PlanToJsonVisitor::PreVisit(SetProperty &op) { json self; self["name"] = "SetProperty"; - self["property"] = ToJson(op.property_, *dba_); + self["property"] = ToJson(op.property_, *request_manager_); self["lhs"] = ToJson(op.lhs_); self["rhs"] = ToJson(op.rhs_); @@ -650,7 +660,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) { json self; self["name"] = "SetLabels"; self["input_symbol"] = ToJson(op.input_symbol_); - self["labels"] = ToJson(op.labels_, *dba_); + self["labels"] = ToJson(op.labels_, *request_manager_); op.input_->Accept(*this); self["input"] = PopOutput(); @@ -662,7 +672,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) { bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) { json self; self["name"] = "RemoveProperty"; - self["property"] = ToJson(op.property_, *dba_); + self["property"] = ToJson(op.property_, *request_manager_); self["lhs"] = ToJson(op.lhs_); op.input_->Accept(*this); @@ -676,7 +686,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) { json self; self["name"] = "RemoveLabels"; self["input_symbol"] = ToJson(op.input_symbol_); - self["labels"] = ToJson(op.labels_, *dba_); + self["labels"] = ToJson(op.labels_, *request_manager_); op.input_->Accept(*this); self["input"] = PopOutput(); diff --git a/src/query/v2/plan/pretty_print.hpp b/src/query/v2/plan/pretty_print.hpp index 9d819606a..c7442b196 100644 --- a/src/query/v2/plan/pretty_print.hpp +++ b/src/query/v2/plan/pretty_print.hpp @@ -18,28 +18,29 @@ #include "query/v2/frontend/ast/ast.hpp" #include "query/v2/plan/operator.hpp" +#include "query/v2/shard_request_manager.hpp" namespace memgraph::query::v2 { -class DbAccessor; namespace plan { class LogicalOperator; /// Pretty print a `LogicalOperator` plan to a `std::ostream`. -/// DbAccessor is needed for resolving label and property names. +/// ShardRequestManager is needed for resolving label and property names. /// Note that `plan_root` isn't modified, but we can't take it as a const /// because we don't have support for visiting a const LogicalOperator. -void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out); +void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root, + std::ostream *out); /// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`. -inline void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root) { - PrettyPrint(dba, plan_root, &std::cout); +inline void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) { + PrettyPrint(request_manager, plan_root, &std::cout); } /// Convert a `LogicalOperator` plan to a JSON representation. /// DbAccessor is needed for resolving label and property names. -nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root); +nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root); class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { public: @@ -47,7 +48,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { using HierarchicalLogicalOperatorVisitor::PreVisit; using HierarchicalLogicalOperatorVisitor::Visit; - PlanPrinter(const DbAccessor *dba, std::ostream *out); + PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out); bool DefaultPreVisit() override; @@ -114,7 +115,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { void Branch(LogicalOperator &op, const std::string &branch_name = ""); int64_t depth_{0}; - const DbAccessor *dba_{nullptr}; + const msgs::ShardRequestManagerInterface *request_manager_{nullptr}; std::ostream *out_{nullptr}; }; @@ -132,20 +133,20 @@ nlohmann::json ToJson(const utils::Bound &bound); nlohmann::json ToJson(const Symbol &symbol); -nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba); +nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager); -nlohmann::json ToJson(storage::v3::LabelId label, const DbAccessor &dba); +nlohmann::json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager); -nlohmann::json ToJson(storage::v3::PropertyId property, const DbAccessor &dba); +nlohmann::json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager); nlohmann::json ToJson(NamedExpression *nexpr); nlohmann::json ToJson(const std::vector> &properties, - const DbAccessor &dba); + const msgs::ShardRequestManagerInterface &request_manager); -nlohmann::json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba); +nlohmann::json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager); -nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba); +nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager); nlohmann::json ToJson(const Aggregate::Element &elem); @@ -160,7 +161,8 @@ nlohmann::json ToJson(const std::vector &items, Args &&...args) { class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor { public: - explicit PlanToJsonVisitor(const DbAccessor *dba) : dba_(dba) {} + explicit PlanToJsonVisitor(const msgs::ShardRequestManagerInterface *request_manager) + : request_manager_(request_manager) {} using HierarchicalLogicalOperatorVisitor::PostVisit; using HierarchicalLogicalOperatorVisitor::PreVisit; @@ -216,7 +218,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor { protected: nlohmann::json output_; - const DbAccessor *dba_; + const msgs::ShardRequestManagerInterface *request_manager_; nlohmann::json PopOutput() { nlohmann::json tmp; diff --git a/src/query/v2/plan/vertex_count_cache.hpp b/src/query/v2/plan/vertex_count_cache.hpp index 72b1dd974..0a02b0ec6 100644 --- a/src/query/v2/plan/vertex_count_cache.hpp +++ b/src/query/v2/plan/vertex_count_cache.hpp @@ -15,6 +15,7 @@ #include #include "query/v2/bindings/typed_value.hpp" +#include "query/v2/shard_request_manager.hpp" #include "storage/v3/conversions.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/property_value.hpp" @@ -28,110 +29,37 @@ namespace memgraph::query::v2::plan { template class VertexCountCache { public: - VertexCountCache(TDbAccessor *db) : db_(db) {} + explicit VertexCountCache(TDbAccessor *shard_request_manager) : shard_request_manager_{shard_request_manager} {} - auto NameToLabel(const std::string &name) { return db_->NameToLabel(name); } - auto NameToProperty(const std::string &name) { return db_->NameToProperty(name); } - auto NameToEdgeType(const std::string &name) { return db_->NameToEdgeType(name); } - - int64_t VerticesCount() { - if (!vertices_count_) vertices_count_ = db_->VerticesCount(); - return *vertices_count_; + auto NameToLabel(const std::string &name) { return shard_request_manager_->LabelNameToLabelId(name); } + auto NameToProperty(const std::string &name) { return shard_request_manager_->NameToProperty(name); } + auto NameToEdgeType(const std::string & /*name*/) { + MG_ASSERT(false, "NameToEdgeType"); + return storage::v3::EdgeTypeId::FromInt(0); } - int64_t VerticesCount(storage::v3::LabelId label) { - if (label_vertex_count_.find(label) == label_vertex_count_.end()) - label_vertex_count_[label] = db_->VerticesCount(label); - return label_vertex_count_.at(label); + int64_t VerticesCount() { return 1; } + + int64_t VerticesCount(storage::v3::LabelId /*label*/) { return 1; } + + int64_t VerticesCount(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return 1; } + + int64_t VerticesCount(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/, + const storage::v3::PropertyValue & /*value*/) { + return 1; } - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) { - auto key = std::make_pair(label, property); - if (label_property_vertex_count_.find(key) == label_property_vertex_count_.end()) - label_property_vertex_count_[key] = db_->VerticesCount(label, property); - return label_property_vertex_count_.at(key); + int64_t VerticesCount(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/, + const std::optional> & /*lower*/, + const std::optional> & /*upper*/) { + return 1; } - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, - const storage::v3::PropertyValue &value) { - auto label_prop = std::make_pair(label, property); - auto &value_vertex_count = property_value_vertex_count_[label_prop]; - // TODO: Why do we even need TypedValue in this whole file? - auto tv_value(storage::v3::PropertyToTypedValue(value)); - if (value_vertex_count.find(tv_value) == value_vertex_count.end()) - value_vertex_count[tv_value] = db_->VerticesCount(label, property, value); - return value_vertex_count.at(tv_value); - } + bool LabelIndexExists(storage::v3::LabelId /*label*/) { return false; } - int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, - const std::optional> &lower, - const std::optional> &upper) { - auto label_prop = std::make_pair(label, property); - auto &bounds_vertex_count = property_bounds_vertex_count_[label_prop]; - BoundsKey bounds = std::make_pair(lower, upper); - if (bounds_vertex_count.find(bounds) == bounds_vertex_count.end()) - bounds_vertex_count[bounds] = db_->VerticesCount(label, property, lower, upper); - return bounds_vertex_count.at(bounds); - } + bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; } - bool LabelIndexExists(storage::v3::LabelId label) { return db_->LabelIndexExists(label); } - - bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId property) { - return db_->LabelPropertyIndexExists(label, property); - } - - private: - typedef std::pair LabelPropertyKey; - - struct LabelPropertyHash { - size_t operator()(const LabelPropertyKey &key) const { - return utils::HashCombine{}(key.first, key.second); - } - }; - - typedef std::pair>, - std::optional>> - BoundsKey; - - struct BoundsHash { - size_t operator()(const BoundsKey &key) const { - const auto &maybe_lower = key.first; - const auto &maybe_upper = key.second; - query::v2::TypedValue lower; - query::v2::TypedValue upper; - if (maybe_lower) lower = storage::v3::PropertyToTypedValue(maybe_lower->value()); - if (maybe_upper) upper = storage::v3::PropertyToTypedValue(maybe_upper->value()); - query::v2::TypedValue::Hash hash; - return utils::HashCombine{}(hash(lower), hash(upper)); - } - }; - - struct BoundsEqual { - bool operator()(const BoundsKey &a, const BoundsKey &b) const { - auto bound_equal = [](const auto &maybe_bound_a, const auto &maybe_bound_b) { - if (maybe_bound_a && maybe_bound_b && maybe_bound_a->type() != maybe_bound_b->type()) return false; - query::v2::TypedValue bound_a; - query::v2::TypedValue bound_b; - if (maybe_bound_a) bound_a = storage::v3::PropertyToTypedValue(maybe_bound_a->value()); - if (maybe_bound_b) bound_b = storage::v3::PropertyToTypedValue(maybe_bound_b->value()); - return query::v2::TypedValue::BoolEqual{}(bound_a, bound_b); - }; - return bound_equal(a.first, b.first) && bound_equal(a.second, b.second); - } - }; - - TDbAccessor *db_; - std::optional vertices_count_; - std::unordered_map label_vertex_count_; - std::unordered_map label_property_vertex_count_; - std::unordered_map< - LabelPropertyKey, - std::unordered_map, - LabelPropertyHash> - property_value_vertex_count_; - std::unordered_map, - LabelPropertyHash> - property_bounds_vertex_count_; + msgs::ShardRequestManagerInterface *shard_request_manager_; }; template diff --git a/src/query/v2/procedure/cypher_type_ptr.hpp b/src/query/v2/procedure/cypher_type_ptr.hpp deleted file mode 100644 index be8d5ff6e..000000000 --- a/src/query/v2/procedure/cypher_type_ptr.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include - -namespace memgraph::query::v2::procedure { -class CypherType; -using CypherTypePtr = std::unique_ptr>; -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/cypher_types.hpp b/src/query/v2/procedure/cypher_types.hpp deleted file mode 100644 index e1f2f1592..000000000 --- a/src/query/v2/procedure/cypher_types.hpp +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/// @file -#pragma once - -#include "mg_procedure.h" - -#include -#include -#include - -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/procedure/cypher_type_ptr.hpp" -#include "query/v2/procedure/mg_procedure_impl.hpp" -#include "utils/memory.hpp" -#include "utils/pmr/string.hpp" - -namespace memgraph::query::v2::procedure { - -class ListType; -class NullableType; - -/// Interface for all supported types in openCypher type system. -class CypherType { - public: - CypherType() = default; - virtual ~CypherType() = default; - - CypherType(const CypherType &) = delete; - CypherType(CypherType &&) = delete; - CypherType &operator=(const CypherType &) = delete; - CypherType &operator=(CypherType &&) = delete; - - /// Get name of the type as it should be presented to the user. - virtual std::string_view GetPresentableName() const = 0; - - /// Return true if given mgp_value is of the type as described by `this`. - virtual bool SatisfiesType(const mgp_value &) const = 0; - - /// Return true if given TypedValue is of the type as described by `this`. - virtual bool SatisfiesType(const query::v2::TypedValue &) const = 0; - - // The following methods are a simple replacement for RTTI because we have - // some special cases we need to handle. - virtual const ListType *AsListType() const { return nullptr; } - virtual const NullableType *AsNullableType() const { return nullptr; } -}; - -// Simple Types - -class AnyType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "ANY"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type != MGP_VALUE_TYPE_NULL; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return !value.IsNull(); } -}; - -class BoolType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "BOOLEAN"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_BOOL; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsBool(); } -}; - -class StringType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "STRING"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_STRING; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsString(); } -}; - -class IntType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "INTEGER"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_INT; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsInt(); } -}; - -class FloatType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "FLOAT"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_DOUBLE; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsDouble(); } -}; - -class NumberType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "NUMBER"; } - - bool SatisfiesType(const mgp_value &value) const override { - return value.type == MGP_VALUE_TYPE_INT || value.type == MGP_VALUE_TYPE_DOUBLE; - } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsInt() || value.IsDouble(); } -}; - -class NodeType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "NODE"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_VERTEX; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsVertex(); } -}; - -class RelationshipType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "RELATIONSHIP"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_EDGE; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsEdge(); } -}; - -class PathType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "PATH"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_PATH; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsPath(); } -}; - -// You'd think that MapType would be a composite type like ListType, but nope. -// Why? No-one really knows. It's defined like that in "CIP2015-09-16 Public -// Type System and Type Annotations" -// Additionally, MapType also covers NodeType and RelationshipType because -// values of that type have property *maps*. -class MapType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "MAP"; } - - bool SatisfiesType(const mgp_value &value) const override { - return value.type == MGP_VALUE_TYPE_MAP || value.type == MGP_VALUE_TYPE_VERTEX || value.type == MGP_VALUE_TYPE_EDGE; - } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { - return value.IsMap() || value.IsVertex() || value.IsEdge(); - } -}; - -// Temporal Types - -class DateType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "DATE"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_DATE; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsDate(); } -}; - -class LocalTimeType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "LOCAL_TIME"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_LOCAL_TIME; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsLocalTime(); } -}; - -class LocalDateTimeType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "LOCAL_DATE_TIME"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_LOCAL_DATE_TIME; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsLocalDateTime(); } -}; - -class DurationType : public CypherType { - public: - std::string_view GetPresentableName() const override { return "DURATION"; } - - bool SatisfiesType(const mgp_value &value) const override { return value.type == MGP_VALUE_TYPE_DURATION; } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { return value.IsDuration(); } -}; - -// Composite Types - -class ListType : public CypherType { - public: - CypherTypePtr element_type_; - utils::pmr::string presentable_name_; - - /// @throw std::bad_alloc - /// @throw std::length_error - explicit ListType(CypherTypePtr element_type, utils::MemoryResource *memory) - : element_type_(std::move(element_type)), presentable_name_("LIST OF ", memory) { - presentable_name_.append(element_type_->GetPresentableName()); - } - - std::string_view GetPresentableName() const override { return presentable_name_; } - - bool SatisfiesType(const mgp_value &value) const override { - if (value.type != MGP_VALUE_TYPE_LIST) { - return false; - } - auto *list = value.list_v; - const auto list_size = list->elems.size(); - for (size_t i = 0; i < list_size; ++i) { - if (!element_type_->SatisfiesType(list->elems[i])) { - return false; - }; - } - return true; - } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { - if (!value.IsList()) return false; - for (const auto &elem : value.ValueList()) { - if (!element_type_->SatisfiesType(elem)) return false; - } - return true; - } - - const ListType *AsListType() const override { return this; } -}; - -class NullableType : public CypherType { - CypherTypePtr type_; - utils::pmr::string presentable_name_; - - // Constructor is private, because we use a factory method Create to prevent - // nesting NullableType on top of each other. - // @throw std::bad_alloc - // @throw std::length_error - explicit NullableType(CypherTypePtr type, utils::MemoryResource *memory) - : type_(std::move(type)), presentable_name_(memory) { - const auto *list_type = type_->AsListType(); - // ListType is specially formatted - if (list_type) { - presentable_name_.assign("LIST? OF ").append(list_type->element_type_->GetPresentableName()); - } else { - presentable_name_.assign(type_->GetPresentableName()).append("?"); - } - } - - public: - /// Create a NullableType of some CypherType. - /// If passed in `type` is already a NullableType, it is returned intact. - /// Otherwise, `type` is wrapped in a new instance of NullableType. - /// @throw std::bad_alloc - /// @throw std::length_error - static CypherTypePtr Create(CypherTypePtr type, utils::MemoryResource *memory) { - if (type->AsNullableType()) return type; - utils::Allocator alloc(memory); - auto *nullable = alloc.allocate(1); - try { - new (nullable) NullableType(std::move(type), memory); - } catch (...) { - alloc.deallocate(nullable, 1); - throw; - } - return CypherTypePtr(nullable, [alloc](CypherType *base_ptr) mutable { - alloc.delete_object(static_cast(base_ptr)); - }); - } - - std::string_view GetPresentableName() const override { return presentable_name_; } - - bool SatisfiesType(const mgp_value &value) const override { - return value.type == MGP_VALUE_TYPE_NULL || type_->SatisfiesType(value); - } - - bool SatisfiesType(const query::v2::TypedValue &value) const override { - return value.IsNull() || type_->SatisfiesType(value); - } - - const NullableType *AsNullableType() const override { return this; } -}; - -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/mg_procedure_helpers.cpp b/src/query/v2/procedure/mg_procedure_helpers.cpp deleted file mode 100644 index f3a624736..000000000 --- a/src/query/v2/procedure/mg_procedure_helpers.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/procedure/mg_procedure_helpers.hpp" - -namespace memgraph::query::v2::procedure { -MgpUniquePtr GetStringValueOrSetError(const char *string, mgp_memory *memory, mgp_result *result) { - procedure::MgpUniquePtr value{nullptr, mgp_value_destroy}; - const auto success = - TryOrSetError([&] { return procedure::CreateMgpObject(value, mgp_value_make_string, string, memory); }, result); - if (!success) { - value.reset(); - } - - return value; -} - -bool InsertResultOrSetError(mgp_result *result, mgp_result_record *record, const char *result_name, mgp_value *value) { - if (const auto err = mgp_result_record_insert(record, result_name, value); err != mgp_error::MGP_ERROR_NO_ERROR) { - const auto error_msg = fmt::format("Unable to set the result for {}, error = {}", result_name, err); - static_cast(mgp_result_set_error_msg(result, error_msg.c_str())); - return false; - } - - return true; -} - -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/mg_procedure_helpers.hpp b/src/query/v2/procedure/mg_procedure_helpers.hpp deleted file mode 100644 index 35248b794..000000000 --- a/src/query/v2/procedure/mg_procedure_helpers.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -#include - -#include "mg_procedure.h" - -namespace memgraph::query::v2::procedure { -template -TResult Call(TFunc func, TArgs... args) { - static_assert(std::is_trivially_copyable_v); - static_assert((std::is_trivially_copyable_v> && ...)); - TResult result{}; - MG_ASSERT(func(args..., &result) == mgp_error::MGP_ERROR_NO_ERROR); - return result; -} - -template -bool CallBool(TFunc func, TArgs... args) { - return Call(func, args...) != 0; -} - -template -using MgpRawObjectDeleter = void (*)(TObj *); - -template -using MgpUniquePtr = std::unique_ptr>; - -template -mgp_error CreateMgpObject(MgpUniquePtr &obj, TFunc func, TArgs &&...args) { - TObj *raw_obj{nullptr}; - const auto err = func(std::forward(args)..., &raw_obj); - obj.reset(raw_obj); - return err; -} - -template -[[nodiscard]] bool TryOrSetError(Fun &&func, mgp_result *result) { - if (const auto err = func(); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - static_cast(mgp_result_set_error_msg(result, "Not enough memory!")); - return false; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - const auto error_msg = fmt::format("Unexpected error ({})!", err); - static_cast(mgp_result_set_error_msg(result, error_msg.c_str())); - return false; - } - return true; -} - -[[nodiscard]] MgpUniquePtr GetStringValueOrSetError(const char *string, mgp_memory *memory, - mgp_result *result); - -[[nodiscard]] bool InsertResultOrSetError(mgp_result *result, mgp_result_record *record, const char *result_name, - mgp_value *value); -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/mg_procedure_impl.cpp b/src/query/v2/procedure/mg_procedure_impl.cpp deleted file mode 100644 index 349ba7da1..000000000 --- a/src/query/v2/procedure/mg_procedure_impl.cpp +++ /dev/null @@ -1,2812 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/procedure/mg_procedure_impl.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mg_procedure.h" -#include "module.hpp" -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/procedure/cypher_types.hpp" -#include "query/v2/procedure/mg_procedure_helpers.hpp" -#include "query/v2/stream/common.hpp" -#include "storage/v3/conversions.hpp" -#include "storage/v3/id_types.hpp" -#include "storage/v3/property_value.hpp" -#include "storage/v3/view.hpp" -#include "utils/algorithm.hpp" -#include "utils/concepts.hpp" -#include "utils/logging.hpp" -#include "utils/math.hpp" -#include "utils/memory.hpp" -#include "utils/string.hpp" -#include "utils/temporal.hpp" -#include "utils/variant_helpers.hpp" - -// This file contains implementation of top level C API functions, but this is -// all actually part of memgraph::query::v2::procedure. So use that namespace for simplicity. -// NOLINTNEXTLINE(google-build-using-namespace) -using namespace memgraph::query::v2::procedure; - -namespace { - -void *MgpAlignedAllocImpl(memgraph::utils::MemoryResource &memory, const size_t size_in_bytes, const size_t alignment) { - if (size_in_bytes == 0U || !memgraph::utils::IsPow2(alignment)) return nullptr; - // Simplify alignment by always using values greater or equal to max_align. - const size_t alloc_align = std::max(alignment, alignof(std::max_align_t)); - // Allocate space for header containing size & alignment info. - const size_t header_size = sizeof(size_in_bytes) + sizeof(alloc_align); - // We need to return the `data` pointer aligned to the requested alignment. - // Since we request the initial memory to be aligned to `alloc_align`, we can - // just allocate an additional multiple of `alloc_align` of bytes such that - // the header fits. `data` will then be aligned after this multiple of bytes. - static_assert(std::is_same_v); - const auto maybe_bytes_for_header = memgraph::utils::RoundUint64ToMultiple(header_size, alloc_align); - if (!maybe_bytes_for_header) return nullptr; - const size_t bytes_for_header = *maybe_bytes_for_header; - const size_t alloc_size = bytes_for_header + size_in_bytes; - if (alloc_size < size_in_bytes) return nullptr; - - void *ptr = memory.Allocate(alloc_size, alloc_align); - char *data = reinterpret_cast(ptr) + bytes_for_header; - std::memcpy(data - sizeof(size_in_bytes), &size_in_bytes, sizeof(size_in_bytes)); - std::memcpy(data - sizeof(size_in_bytes) - sizeof(alloc_align), &alloc_align, sizeof(alloc_align)); - return data; -} - -void MgpFreeImpl(memgraph::utils::MemoryResource &memory, void *const p) noexcept { - try { - if (!p) return; - char *const data = reinterpret_cast(p); - // Read the header containing size & alignment info. - size_t size_in_bytes{}; - std::memcpy(&size_in_bytes, data - sizeof(size_in_bytes), sizeof(size_in_bytes)); - size_t alloc_align{}; - std::memcpy(&alloc_align, data - sizeof(size_in_bytes) - sizeof(alloc_align), sizeof(alloc_align)); - // Reconstruct how many bytes we allocated on top of the original request. - // We need not check allocation request overflow, since we did so already in - // mgp_aligned_alloc. - const size_t header_size = sizeof(size_in_bytes) + sizeof(alloc_align); - const size_t bytes_for_header = *memgraph::utils::RoundUint64ToMultiple(header_size, alloc_align); - const size_t alloc_size = bytes_for_header + size_in_bytes; - // Get the original ptr we allocated. - void *const original_ptr = data - bytes_for_header; - memory.Deallocate(original_ptr, alloc_size, alloc_align); - } catch (const memgraph::utils::BasicException &be) { - spdlog::error("BasicException during the release of memory for query modules: {}", be.what()); - } catch (const std::exception &e) { - spdlog::error("std::exception during the release of memory for query modules: {}", e.what()); - } catch (...) { - spdlog::error("Unexpected throw during the release of memory for query modules"); - } -} -struct DeletedObjectException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -struct KeyAlreadyExistsException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -struct InsufficientBufferException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -struct ImmutableObjectException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -struct ValueConversionException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -struct SerializationException : public memgraph::utils::BasicException { - using memgraph::utils::BasicException::BasicException; -}; - -template -concept ReturnsType = std::same_as, TReturn>; - -template -concept ReturnsVoid = ReturnsType; - -template -void WrapExceptionsHelper(TFunc &&func) { - std::forward(func)(); -} - -template > -void WrapExceptionsHelper(TFunc &&func, TReturn *result) { - *result = {}; - *result = std::forward(func)(); -} - -template -[[nodiscard]] mgp_error WrapExceptions(TFunc &&func, Args &&...args) noexcept { - static_assert(sizeof...(args) <= 1, "WrapExceptions should have only one or zero parameter!"); - try { - WrapExceptionsHelper(std::forward(func), std::forward(args)...); - } catch (const DeletedObjectException &neoe) { - spdlog::error("Deleted object error during mg API call: {}", neoe.what()); - return mgp_error::MGP_ERROR_DELETED_OBJECT; - } catch (const KeyAlreadyExistsException &kaee) { - spdlog::error("Key already exists error during mg API call: {}", kaee.what()); - return mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS; - } catch (const InsufficientBufferException &ibe) { - spdlog::error("Insufficient buffer error during mg API call: {}", ibe.what()); - return mgp_error::MGP_ERROR_INSUFFICIENT_BUFFER; - } catch (const ImmutableObjectException &ioe) { - spdlog::error("Immutable object error during mg API call: {}", ioe.what()); - return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT; - } catch (const ValueConversionException &vce) { - spdlog::error("Value converion error during mg API call: {}", vce.what()); - return mgp_error::MGP_ERROR_VALUE_CONVERSION; - } catch (const SerializationException &se) { - spdlog::error("Serialization error during mg API call: {}", se.what()); - return mgp_error::MGP_ERROR_SERIALIZATION_ERROR; - } catch (const std::bad_alloc &bae) { - spdlog::error("Memory allocation error during mg API call: {}", bae.what()); - return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE; - } catch (const memgraph::utils::OutOfMemoryException &oome) { - spdlog::error("Memory limit exceeded during mg API call: {}", oome.what()); - return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE; - } catch (const std::out_of_range &oore) { - spdlog::error("Out of range error during mg API call: {}", oore.what()); - return mgp_error::MGP_ERROR_OUT_OF_RANGE; - } catch (const std::invalid_argument &iae) { - spdlog::error("Invalid argument error during mg API call: {}", iae.what()); - return mgp_error::MGP_ERROR_INVALID_ARGUMENT; - } catch (const std::logic_error &lee) { - spdlog::error("Logic error during mg API call: {}", lee.what()); - return mgp_error::MGP_ERROR_LOGIC_ERROR; - } catch (const std::exception &e) { - spdlog::error("Unexpected error during mg API call: {}", e.what()); - return mgp_error::MGP_ERROR_UNKNOWN_ERROR; - } catch (const memgraph::utils::temporal::InvalidArgumentException &e) { - spdlog::error("Invalid argument was sent to an mg API call for temporal types: {}", e.what()); - return mgp_error::MGP_ERROR_INVALID_ARGUMENT; - } catch (...) { - spdlog::error("Unexpected error during mg API call"); - return mgp_error::MGP_ERROR_UNKNOWN_ERROR; - } - return mgp_error::MGP_ERROR_NO_ERROR; -} - -// Graph mutations -bool MgpGraphIsMutable(const mgp_graph &graph) noexcept { - return graph.view == memgraph::storage::v3::View::NEW && graph.ctx != nullptr; -} - -bool MgpVertexIsMutable(const mgp_vertex &vertex) { return MgpGraphIsMutable(*vertex.graph); } - -bool MgpEdgeIsMutable(const mgp_edge &edge) { return MgpVertexIsMutable(edge.from); } -} // namespace - -mgp_error mgp_alloc(mgp_memory *memory, size_t size_in_bytes, void **result) { - return mgp_aligned_alloc(memory, size_in_bytes, alignof(std::max_align_t), result); -} - -mgp_error mgp_aligned_alloc(mgp_memory *memory, const size_t size_in_bytes, const size_t alignment, void **result) { - return WrapExceptions( - [memory, size_in_bytes, alignment] { return MgpAlignedAllocImpl(*memory->impl, size_in_bytes, alignment); }, - result); -} - -void mgp_free(mgp_memory *memory, void *const p) { - static_assert(noexcept(MgpFreeImpl(*memory->impl, p))); - MgpFreeImpl(*memory->impl, p); -} - -mgp_error mgp_global_alloc(size_t size_in_bytes, void **result) { - return mgp_global_aligned_alloc(size_in_bytes, alignof(std::max_align_t), result); -} - -mgp_error mgp_global_aligned_alloc(size_t size_in_bytes, size_t alignment, void **result) { - return WrapExceptions( - [size_in_bytes, alignment] { - return MgpAlignedAllocImpl(gModuleRegistry.GetSharedMemoryResource(), size_in_bytes, alignment); - }, - result); -} - -void mgp_global_free(void *const p) { - static_assert(noexcept(MgpFreeImpl(gModuleRegistry.GetSharedMemoryResource(), p))); - MgpFreeImpl(gModuleRegistry.GetSharedMemoryResource(), p); -} - -namespace { - -template -U *NewRawMgpObject(memgraph::utils::MemoryResource *memory, TArgs &&...args) { - memgraph::utils::Allocator allocator(memory); - return allocator.template new_object(std::forward(args)...); -} - -template -U *NewRawMgpObject(mgp_memory *memory, TArgs &&...args) { - return NewRawMgpObject(memory->impl, std::forward(args)...); -} - -// Assume that deallocation and object destruction never throws. If it does, -// we are in big trouble. -template -void DeleteRawMgpObject(T *ptr) noexcept { - try { - if (!ptr) return; - memgraph::utils::Allocator allocator(ptr->GetMemoryResource()); - allocator.delete_object(ptr); - } catch (...) { - LOG_FATAL("Cannot deallocate mgp object"); - } -} - -template -MgpUniquePtr NewMgpObject(mgp_memory *memory, TArgs &&...args) { - return MgpUniquePtr(NewRawMgpObject(memory->impl, std::forward(args)...), &DeleteRawMgpObject); -} - -mgp_value_type FromTypedValueType(memgraph::query::v2::TypedValue::Type type) { - switch (type) { - case memgraph::query::v2::TypedValue::Type::Null: - return MGP_VALUE_TYPE_NULL; - case memgraph::query::v2::TypedValue::Type::Bool: - return MGP_VALUE_TYPE_BOOL; - case memgraph::query::v2::TypedValue::Type::Int: - return MGP_VALUE_TYPE_INT; - case memgraph::query::v2::TypedValue::Type::Double: - return MGP_VALUE_TYPE_DOUBLE; - case memgraph::query::v2::TypedValue::Type::String: - return MGP_VALUE_TYPE_STRING; - case memgraph::query::v2::TypedValue::Type::List: - return MGP_VALUE_TYPE_LIST; - case memgraph::query::v2::TypedValue::Type::Map: - return MGP_VALUE_TYPE_MAP; - case memgraph::query::v2::TypedValue::Type::Vertex: - return MGP_VALUE_TYPE_VERTEX; - case memgraph::query::v2::TypedValue::Type::Edge: - return MGP_VALUE_TYPE_EDGE; - case memgraph::query::v2::TypedValue::Type::Path: - return MGP_VALUE_TYPE_PATH; - case memgraph::query::v2::TypedValue::Type::Date: - return MGP_VALUE_TYPE_DATE; - case memgraph::query::v2::TypedValue::Type::LocalTime: - return MGP_VALUE_TYPE_LOCAL_TIME; - case memgraph::query::v2::TypedValue::Type::LocalDateTime: - return MGP_VALUE_TYPE_LOCAL_DATE_TIME; - case memgraph::query::v2::TypedValue::Type::Duration: - return MGP_VALUE_TYPE_DURATION; - } -} -} // namespace - -memgraph::query::v2::TypedValue ToTypedValue(const mgp_value &val, memgraph::utils::MemoryResource *memory) { - switch (val.type) { - case MGP_VALUE_TYPE_NULL: - return memgraph::query::v2::TypedValue(memory); - case MGP_VALUE_TYPE_BOOL: - return memgraph::query::v2::TypedValue(val.bool_v, memory); - case MGP_VALUE_TYPE_INT: - return memgraph::query::v2::TypedValue(val.int_v, memory); - case MGP_VALUE_TYPE_DOUBLE: - return memgraph::query::v2::TypedValue(val.double_v, memory); - case MGP_VALUE_TYPE_STRING: - return {val.string_v, memory}; - case MGP_VALUE_TYPE_LIST: { - const auto *list = val.list_v; - memgraph::query::v2::TypedValue::TVector tv_list(memory); - tv_list.reserve(list->elems.size()); - for (const auto &elem : list->elems) { - tv_list.emplace_back(ToTypedValue(elem, memory)); - } - return memgraph::query::v2::TypedValue(std::move(tv_list)); - } - case MGP_VALUE_TYPE_MAP: { - const auto *map = val.map_v; - memgraph::query::v2::TypedValue::TMap tv_map(memory); - for (const auto &item : map->items) { - tv_map.emplace(item.first, ToTypedValue(item.second, memory)); - } - return memgraph::query::v2::TypedValue(std::move(tv_map)); - } - case MGP_VALUE_TYPE_VERTEX: - return memgraph::query::v2::TypedValue(val.vertex_v->impl, memory); - case MGP_VALUE_TYPE_EDGE: - return memgraph::query::v2::TypedValue(val.edge_v->impl, memory); - case MGP_VALUE_TYPE_PATH: { - const auto *path = val.path_v; - MG_ASSERT(!path->vertices.empty()); - MG_ASSERT(path->vertices.size() == path->edges.size() + 1); - memgraph::query::v2::Path tv_path(path->vertices[0].impl, memory); - for (size_t i = 0; i < path->edges.size(); ++i) { - tv_path.Expand(path->edges[i].impl); - tv_path.Expand(path->vertices[i + 1].impl); - } - return memgraph::query::v2::TypedValue(std::move(tv_path)); - } - case MGP_VALUE_TYPE_DATE: - return memgraph::query::v2::TypedValue(val.date_v->date, memory); - case MGP_VALUE_TYPE_LOCAL_TIME: - return memgraph::query::v2::TypedValue(val.local_time_v->local_time, memory); - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: - return memgraph::query::v2::TypedValue(val.local_date_time_v->local_date_time, memory); - case MGP_VALUE_TYPE_DURATION: - return memgraph::query::v2::TypedValue(val.duration_v->duration, memory); - } -} - -mgp_value::mgp_value(memgraph::utils::MemoryResource *m) noexcept : type(MGP_VALUE_TYPE_NULL), memory(m) {} - -mgp_value::mgp_value(bool val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_BOOL), memory(m), bool_v(val) {} - -mgp_value::mgp_value(int64_t val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_INT), memory(m), int_v(val) {} - -mgp_value::mgp_value(double val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_DOUBLE), memory(m), double_v(val) {} - -mgp_value::mgp_value(const char *val, memgraph::utils::MemoryResource *m) - : type(MGP_VALUE_TYPE_STRING), memory(m), string_v(val, m) {} - -mgp_value::mgp_value(mgp_list *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_LIST), memory(m), list_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_map *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_MAP), memory(m), map_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_vertex *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_VERTEX), memory(m), vertex_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_edge *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_EDGE), memory(m), edge_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_path *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_PATH), memory(m), path_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_date *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_DATE), memory(m), date_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_local_time *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_LOCAL_TIME), memory(m), local_time_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_local_date_time *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_LOCAL_DATE_TIME), memory(m), local_date_time_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(mgp_duration *val, memgraph::utils::MemoryResource *m) noexcept - : type(MGP_VALUE_TYPE_DURATION), memory(m), duration_v(val) { - MG_ASSERT(val->GetMemoryResource() == m, "Unable to take ownership of a pointer with different allocator."); -} - -mgp_value::mgp_value(const memgraph::query::v2::TypedValue &tv, mgp_graph *graph, memgraph::utils::MemoryResource *m) - : type(FromTypedValueType(tv.type())), memory(m) { - switch (type) { - case MGP_VALUE_TYPE_NULL: - break; - case MGP_VALUE_TYPE_BOOL: - bool_v = tv.ValueBool(); - break; - case MGP_VALUE_TYPE_INT: - int_v = tv.ValueInt(); - break; - case MGP_VALUE_TYPE_DOUBLE: - double_v = tv.ValueDouble(); - break; - case MGP_VALUE_TYPE_STRING: - new (&string_v) memgraph::utils::pmr::string(tv.ValueString(), m); - break; - case MGP_VALUE_TYPE_LIST: { - // Fill the stack allocated container and then construct the actual member - // value. This handles the case when filling the container throws - // something and our destructor doesn't get called so member value isn't - // released. - memgraph::utils::pmr::vector elems(m); - elems.reserve(tv.ValueList().size()); - for (const auto &elem : tv.ValueList()) { - elems.emplace_back(elem, graph); - } - memgraph::utils::Allocator allocator(m); - list_v = allocator.new_object(std::move(elems)); - break; - } - case MGP_VALUE_TYPE_MAP: { - // Fill the stack allocated container and then construct the actual member - // value. This handles the case when filling the container throws - // something and our destructor doesn't get called so member value isn't - // released. - memgraph::utils::pmr::map items(m); - for (const auto &item : tv.ValueMap()) { - items.emplace(item.first, mgp_value(item.second, graph, m)); - } - memgraph::utils::Allocator allocator(m); - map_v = allocator.new_object(std::move(items)); - break; - } - case MGP_VALUE_TYPE_VERTEX: { - memgraph::utils::Allocator allocator(m); - vertex_v = allocator.new_object(tv.ValueVertex(), graph); - break; - } - case MGP_VALUE_TYPE_EDGE: { - memgraph::utils::Allocator allocator(m); - edge_v = allocator.new_object(tv.ValueEdge(), graph); - break; - } - case MGP_VALUE_TYPE_PATH: { - // Fill the stack allocated container and then construct the actual member - // value. This handles the case when filling the container throws - // something and our destructor doesn't get called so member value isn't - // released. - mgp_path tmp_path(m); - tmp_path.vertices.reserve(tv.ValuePath().vertices().size()); - for (const auto &v : tv.ValuePath().vertices()) { - tmp_path.vertices.emplace_back(v, graph); - } - tmp_path.edges.reserve(tv.ValuePath().edges().size()); - for (const auto &e : tv.ValuePath().edges()) { - tmp_path.edges.emplace_back(e, graph); - } - memgraph::utils::Allocator allocator(m); - path_v = allocator.new_object(std::move(tmp_path)); - break; - } - case MGP_VALUE_TYPE_DATE: { - memgraph::utils::Allocator allocator(m); - date_v = allocator.new_object(tv.ValueDate()); - break; - } - case MGP_VALUE_TYPE_LOCAL_TIME: { - memgraph::utils::Allocator allocator(m); - local_time_v = allocator.new_object(tv.ValueLocalTime()); - break; - } - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: { - memgraph::utils::Allocator allocator(m); - local_date_time_v = allocator.new_object(tv.ValueLocalDateTime()); - break; - } - case MGP_VALUE_TYPE_DURATION: { - memgraph::utils::Allocator allocator(m); - duration_v = allocator.new_object(tv.ValueDuration()); - break; - } - } -} - -mgp_value::mgp_value(const memgraph::storage::v3::PropertyValue &pv, memgraph::utils::MemoryResource *m) : memory(m) { - switch (pv.type()) { - case memgraph::storage::v3::PropertyValue::Type::Null: - type = MGP_VALUE_TYPE_NULL; - break; - case memgraph::storage::v3::PropertyValue::Type::Bool: - type = MGP_VALUE_TYPE_BOOL; - bool_v = pv.ValueBool(); - break; - case memgraph::storage::v3::PropertyValue::Type::Int: - type = MGP_VALUE_TYPE_INT; - int_v = pv.ValueInt(); - break; - case memgraph::storage::v3::PropertyValue::Type::Double: - type = MGP_VALUE_TYPE_DOUBLE; - double_v = pv.ValueDouble(); - break; - case memgraph::storage::v3::PropertyValue::Type::String: - type = MGP_VALUE_TYPE_STRING; - new (&string_v) memgraph::utils::pmr::string(pv.ValueString(), m); - break; - case memgraph::storage::v3::PropertyValue::Type::List: { - // Fill the stack allocated container and then construct the actual member - // value. This handles the case when filling the container throws - // something and our destructor doesn't get called so member value isn't - // released. - type = MGP_VALUE_TYPE_LIST; - memgraph::utils::pmr::vector elems(m); - elems.reserve(pv.ValueList().size()); - for (const auto &elem : pv.ValueList()) { - elems.emplace_back(elem); - } - memgraph::utils::Allocator allocator(m); - list_v = allocator.new_object(std::move(elems)); - break; - } - case memgraph::storage::v3::PropertyValue::Type::Map: { - // Fill the stack allocated container and then construct the actual member - // value. This handles the case when filling the container throws - // something and our destructor doesn't get called so member value isn't - // released. - type = MGP_VALUE_TYPE_MAP; - memgraph::utils::pmr::map items(m); - for (const auto &item : pv.ValueMap()) { - items.emplace(item.first, item.second); - } - memgraph::utils::Allocator allocator(m); - map_v = allocator.new_object(std::move(items)); - break; - } - case memgraph::storage::v3::PropertyValue::Type::TemporalData: { - const auto &temporal_data = pv.ValueTemporalData(); - switch (temporal_data.type) { - case memgraph::storage::v3::TemporalType::Date: { - type = MGP_VALUE_TYPE_DATE; - date_v = NewRawMgpObject(m, temporal_data.microseconds); - break; - } - case memgraph::storage::v3::TemporalType::LocalTime: { - type = MGP_VALUE_TYPE_LOCAL_TIME; - local_time_v = NewRawMgpObject(m, temporal_data.microseconds); - break; - } - case memgraph::storage::v3::TemporalType::LocalDateTime: { - type = MGP_VALUE_TYPE_LOCAL_DATE_TIME; - local_date_time_v = NewRawMgpObject(m, temporal_data.microseconds); - break; - } - case memgraph::storage::v3::TemporalType::Duration: { - type = MGP_VALUE_TYPE_DURATION; - duration_v = NewRawMgpObject(m, temporal_data.microseconds); - break; - } - } - } - } -} - -mgp_value::mgp_value(const mgp_value &other, memgraph::utils::MemoryResource *m) : type(other.type), memory(m) { - switch (other.type) { - case MGP_VALUE_TYPE_NULL: - break; - case MGP_VALUE_TYPE_BOOL: - bool_v = other.bool_v; - break; - case MGP_VALUE_TYPE_INT: - int_v = other.int_v; - break; - case MGP_VALUE_TYPE_DOUBLE: - double_v = other.double_v; - break; - case MGP_VALUE_TYPE_STRING: - new (&string_v) memgraph::utils::pmr::string(other.string_v, m); - break; - case MGP_VALUE_TYPE_LIST: { - memgraph::utils::Allocator allocator(m); - list_v = allocator.new_object(*other.list_v); - break; - } - case MGP_VALUE_TYPE_MAP: { - memgraph::utils::Allocator allocator(m); - map_v = allocator.new_object(*other.map_v); - break; - } - case MGP_VALUE_TYPE_VERTEX: { - memgraph::utils::Allocator allocator(m); - vertex_v = allocator.new_object(*other.vertex_v); - break; - } - case MGP_VALUE_TYPE_EDGE: { - memgraph::utils::Allocator allocator(m); - edge_v = allocator.new_object(*other.edge_v); - break; - } - case MGP_VALUE_TYPE_PATH: { - memgraph::utils::Allocator allocator(m); - path_v = allocator.new_object(*other.path_v); - break; - } - case MGP_VALUE_TYPE_DATE: { - date_v = NewRawMgpObject(m, *other.date_v); - break; - } - case MGP_VALUE_TYPE_LOCAL_TIME: { - local_time_v = NewRawMgpObject(m, *other.local_time_v); - break; - } - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: { - local_date_time_v = NewRawMgpObject(m, *other.local_date_time_v); - break; - } - case MGP_VALUE_TYPE_DURATION: { - duration_v = NewRawMgpObject(m, *other.duration_v); - break; - } - } -} - -namespace { - -void DeleteValueMember(mgp_value *value) noexcept { - MG_ASSERT(value); - memgraph::utils::Allocator allocator(value->GetMemoryResource()); - switch (Call(mgp_value_get_type, value)) { - case MGP_VALUE_TYPE_NULL: - case MGP_VALUE_TYPE_BOOL: - case MGP_VALUE_TYPE_INT: - case MGP_VALUE_TYPE_DOUBLE: - return; - case MGP_VALUE_TYPE_STRING: - using TString = memgraph::utils::pmr::string; - value->string_v.~TString(); - return; - case MGP_VALUE_TYPE_LIST: - allocator.delete_object(value->list_v); - return; - case MGP_VALUE_TYPE_MAP: - allocator.delete_object(value->map_v); - return; - case MGP_VALUE_TYPE_VERTEX: - allocator.delete_object(value->vertex_v); - return; - case MGP_VALUE_TYPE_EDGE: - allocator.delete_object(value->edge_v); - return; - case MGP_VALUE_TYPE_PATH: - allocator.delete_object(value->path_v); - return; - case MGP_VALUE_TYPE_DATE: - allocator.delete_object(value->date_v); - return; - case MGP_VALUE_TYPE_LOCAL_TIME: - allocator.delete_object(value->local_time_v); - return; - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: - allocator.delete_object(value->local_date_time_v); - return; - case MGP_VALUE_TYPE_DURATION: - allocator.delete_object(value->duration_v); - return; - } -} - -} // namespace - -mgp_value::mgp_value(mgp_value &&other, memgraph::utils::MemoryResource *m) : type(other.type), memory(m) { - switch (other.type) { - case MGP_VALUE_TYPE_NULL: - break; - case MGP_VALUE_TYPE_BOOL: - bool_v = other.bool_v; - break; - case MGP_VALUE_TYPE_INT: - int_v = other.int_v; - break; - case MGP_VALUE_TYPE_DOUBLE: - double_v = other.double_v; - break; - case MGP_VALUE_TYPE_STRING: - new (&string_v) memgraph::utils::pmr::string(std::move(other.string_v), m); - break; - case MGP_VALUE_TYPE_LIST: - static_assert(std::is_pointer_v, "Expected to move list_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - list_v = other.list_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - memgraph::utils::Allocator allocator(m); - list_v = allocator.new_object(std::move(*other.list_v)); - } - break; - case MGP_VALUE_TYPE_MAP: - static_assert(std::is_pointer_v, "Expected to move map_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - map_v = other.map_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - memgraph::utils::Allocator allocator(m); - map_v = allocator.new_object(std::move(*other.map_v)); - } - break; - case MGP_VALUE_TYPE_VERTEX: - static_assert(std::is_pointer_v, "Expected to move vertex_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - vertex_v = other.vertex_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - memgraph::utils::Allocator allocator(m); - vertex_v = allocator.new_object(std::move(*other.vertex_v)); - } - break; - case MGP_VALUE_TYPE_EDGE: - static_assert(std::is_pointer_v, "Expected to move edge_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - edge_v = other.edge_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - memgraph::utils::Allocator allocator(m); - edge_v = allocator.new_object(std::move(*other.edge_v)); - } - break; - case MGP_VALUE_TYPE_PATH: - static_assert(std::is_pointer_v, "Expected to move path_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - path_v = other.path_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - memgraph::utils::Allocator allocator(m); - path_v = allocator.new_object(std::move(*other.path_v)); - } - break; - case MGP_VALUE_TYPE_DATE: - static_assert(std::is_pointer_v, "Expected to move date_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - date_v = other.date_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - date_v = NewRawMgpObject(m, *other.date_v); - } - break; - case MGP_VALUE_TYPE_LOCAL_TIME: - static_assert(std::is_pointer_v, "Expected to move local_time_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - local_time_v = other.local_time_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - local_time_v = NewRawMgpObject(m, *other.local_time_v); - } - break; - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: - static_assert(std::is_pointer_v, - "Expected to move local_date_time_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - local_date_time_v = other.local_date_time_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - local_date_time_v = NewRawMgpObject(m, *other.local_date_time_v); - } - break; - case MGP_VALUE_TYPE_DURATION: - static_assert(std::is_pointer_v, "Expected to move duration_v by copying pointers."); - if (*other.GetMemoryResource() == *m) { - duration_v = other.duration_v; - other.type = MGP_VALUE_TYPE_NULL; - } else { - duration_v = NewRawMgpObject(m, *other.duration_v); - } - break; - } - DeleteValueMember(&other); - other.type = MGP_VALUE_TYPE_NULL; -} - -mgp_value::~mgp_value() noexcept { DeleteValueMember(this); } - -mgp_edge *mgp_edge::Copy(const mgp_edge &edge, mgp_memory &memory) { - return NewRawMgpObject(&memory, edge.impl, edge.from.graph); -} - -void mgp_value_destroy(mgp_value *val) { DeleteRawMgpObject(val); } - -mgp_error mgp_value_make_null(mgp_memory *memory, mgp_value **result) { - return WrapExceptions([memory] { return NewRawMgpObject(memory); }, result); -} - -mgp_error mgp_value_make_bool(int val, mgp_memory *memory, mgp_value **result) { - return WrapExceptions([val, memory] { return NewRawMgpObject(memory, val != 0); }, result); -} - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(type, param) \ - mgp_error mgp_value_make_##type(param val, mgp_memory *memory, mgp_value **result) { \ - return WrapExceptions([val, memory] { return NewRawMgpObject(memory, val); }, result); \ - } - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(int, int64_t); -DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(double, double); -DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(string, const char *); - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_MGP_VALUE_MAKE(type) \ - mgp_error mgp_value_make_##type(mgp_##type *val, mgp_value **result) { \ - return WrapExceptions([val] { return NewRawMgpObject(val->GetMemoryResource(), val); }, result); \ - } - -DEFINE_MGP_VALUE_MAKE(list) -DEFINE_MGP_VALUE_MAKE(map) -DEFINE_MGP_VALUE_MAKE(vertex) -DEFINE_MGP_VALUE_MAKE(edge) -DEFINE_MGP_VALUE_MAKE(path) -DEFINE_MGP_VALUE_MAKE(date) -DEFINE_MGP_VALUE_MAKE(local_time) -DEFINE_MGP_VALUE_MAKE(local_date_time) -DEFINE_MGP_VALUE_MAKE(duration) - -namespace { -mgp_value_type MgpValueGetType(const mgp_value &val) noexcept { return val.type; } -} // namespace - -mgp_error mgp_value_get_type(mgp_value *val, mgp_value_type *result) { - static_assert(noexcept(MgpValueGetType(*val))); - *result = MgpValueGetType(*val); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_MGP_VALUE_IS(type_lowercase, type_uppercase) \ - mgp_error mgp_value_is_##type_lowercase(mgp_value *val, int *result) { \ - static_assert(noexcept(MgpValueGetType(*val))); \ - *result = MgpValueGetType(*val) == MGP_VALUE_TYPE_##type_uppercase; \ - return mgp_error::MGP_ERROR_NO_ERROR; \ - } - -DEFINE_MGP_VALUE_IS(null, NULL) -DEFINE_MGP_VALUE_IS(bool, BOOL) -DEFINE_MGP_VALUE_IS(int, INT) -DEFINE_MGP_VALUE_IS(double, DOUBLE) -DEFINE_MGP_VALUE_IS(string, STRING) -DEFINE_MGP_VALUE_IS(list, LIST) -DEFINE_MGP_VALUE_IS(map, MAP) -DEFINE_MGP_VALUE_IS(vertex, VERTEX) -DEFINE_MGP_VALUE_IS(edge, EDGE) -DEFINE_MGP_VALUE_IS(path, PATH) -DEFINE_MGP_VALUE_IS(date, DATE) -DEFINE_MGP_VALUE_IS(local_time, LOCAL_TIME) -DEFINE_MGP_VALUE_IS(local_date_time, LOCAL_DATE_TIME) -DEFINE_MGP_VALUE_IS(duration, DURATION) - -mgp_error mgp_value_get_bool(mgp_value *val, int *result) { - *result = val->bool_v ? 1 : 0; - return mgp_error::MGP_ERROR_NO_ERROR; -} -mgp_error mgp_value_get_int(mgp_value *val, int64_t *result) { - *result = val->int_v; - return mgp_error::MGP_ERROR_NO_ERROR; -} -mgp_error mgp_value_get_double(mgp_value *val, double *result) { - *result = val->double_v; - return mgp_error::MGP_ERROR_NO_ERROR; -} -mgp_error mgp_value_get_string(mgp_value *val, const char **result) { - static_assert(noexcept(val->string_v.c_str())); - *result = val->string_v.c_str(); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_MGP_VALUE_GET(type) \ - mgp_error mgp_value_get_##type(mgp_value *val, mgp_##type **result) { \ - *result = val->type##_v; \ - return mgp_error::MGP_ERROR_NO_ERROR; \ - } - -DEFINE_MGP_VALUE_GET(list) -DEFINE_MGP_VALUE_GET(map) -DEFINE_MGP_VALUE_GET(vertex) -DEFINE_MGP_VALUE_GET(edge) -DEFINE_MGP_VALUE_GET(path) -DEFINE_MGP_VALUE_GET(date) -DEFINE_MGP_VALUE_GET(local_time) -DEFINE_MGP_VALUE_GET(local_date_time) -DEFINE_MGP_VALUE_GET(duration) - -mgp_error mgp_list_make_empty(size_t capacity, mgp_memory *memory, mgp_list **result) { - return WrapExceptions( - [capacity, memory] { - auto list = NewMgpObject(memory); - list->elems.reserve(capacity); - return list.release(); - }, - result); -} - -void mgp_list_destroy(mgp_list *list) { DeleteRawMgpObject(list); } - -namespace { -void MgpListAppendExtend(mgp_list &list, const mgp_value &value) { list.elems.push_back(value); } -} // namespace - -mgp_error mgp_list_append(mgp_list *list, mgp_value *val) { - return WrapExceptions([list, val] { - if (Call(mgp_list_size, list) >= Call(mgp_list_capacity, list)) { - throw InsufficientBufferException{ - "Cannot append a new value to the mgp_list without extending it, because its size reached its capacity!"}; - } - MgpListAppendExtend(*list, *val); - }); -} - -mgp_error mgp_list_append_extend(mgp_list *list, mgp_value *val) { - return WrapExceptions([list, val] { MgpListAppendExtend(*list, *val); }); -} - -mgp_error mgp_list_size(mgp_list *list, size_t *result) { - static_assert(noexcept(list->elems.size())); - *result = list->elems.size(); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_list_capacity(mgp_list *list, size_t *result) { - static_assert(noexcept(list->elems.capacity())); - *result = list->elems.capacity(); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_list_at(mgp_list *list, size_t i, mgp_value **result) { - return WrapExceptions( - [list, i] { - if (i >= Call(mgp_list_size, list)) { - throw std::out_of_range("Element cannot be retrieved, because index exceeds list's size!"); - } - return &list->elems[i]; - }, - result); -} - -mgp_error mgp_map_make_empty(mgp_memory *memory, mgp_map **result) { - return WrapExceptions([&memory] { return NewRawMgpObject(memory); }, result); -} - -void mgp_map_destroy(mgp_map *map) { DeleteRawMgpObject(map); } - -mgp_error mgp_map_insert(mgp_map *map, const char *key, mgp_value *value) { - return WrapExceptions([&] { - auto emplace_result = map->items.emplace(key, *value); - if (!emplace_result.second) { - throw KeyAlreadyExistsException{"Map already contains mapping for {}", key}; - } - }); -} - -mgp_error mgp_map_size(mgp_map *map, size_t *result) { - static_assert(noexcept(map->items.size())); - *result = map->items.size(); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_map_at(mgp_map *map, const char *key, mgp_value **result) { - return WrapExceptions( - [&map, &key]() -> mgp_value * { - auto found_it = map->items.find(key); - if (found_it == map->items.end()) { - return nullptr; - }; - return &found_it->second; - }, - result); -} - -mgp_error mgp_map_item_key(mgp_map_item *item, const char **result) { - return WrapExceptions([&item] { return item->key; }, result); -} - -mgp_error mgp_map_item_value(mgp_map_item *item, mgp_value **result) { - return WrapExceptions([item] { return item->value; }, result); -} - -mgp_error mgp_map_iter_items(mgp_map *map, mgp_memory *memory, mgp_map_items_iterator **result) { - return WrapExceptions([map, memory] { return NewRawMgpObject(memory, map); }, result); -} - -void mgp_map_items_iterator_destroy(mgp_map_items_iterator *it) { DeleteRawMgpObject(it); } - -mgp_error mgp_map_items_iterator_get(mgp_map_items_iterator *it, mgp_map_item **result) { - return WrapExceptions( - [it]() -> mgp_map_item * { - if (it->current_it == it->map->items.end()) { - return nullptr; - }; - return &it->current; - }, - result); -} - -mgp_error mgp_map_items_iterator_next(mgp_map_items_iterator *it, mgp_map_item **result) { - return WrapExceptions( - [it]() -> mgp_map_item * { - if (it->current_it == it->map->items.end()) { - return nullptr; - } - if (++it->current_it == it->map->items.end()) { - return nullptr; - } - it->current.key = it->current_it->first.c_str(); - it->current.value = &it->current_it->second; - return &it->current; - }, - result); -} - -mgp_error mgp_path_make_with_start(mgp_vertex *vertex, mgp_memory *memory, mgp_path **result) { - return WrapExceptions( - [vertex, memory]() -> mgp_path * { - auto path = NewMgpObject(memory); - if (path == nullptr) { - return nullptr; - } - path->vertices.push_back(*vertex); - return path.release(); - }, - result); -} - -mgp_error mgp_path_copy(mgp_path *path, mgp_memory *memory, mgp_path **result) { - return WrapExceptions( - [path, memory] { - MG_ASSERT(Call(mgp_path_size, path) == path->vertices.size() - 1, "Invalid mgp_path"); - return NewRawMgpObject(memory, *path); - }, - result); -} - -void mgp_path_destroy(mgp_path *path) { DeleteRawMgpObject(path); } - -mgp_error mgp_path_expand(mgp_path *path, mgp_edge *edge) { - return WrapExceptions([path, edge] { - MG_ASSERT(Call(mgp_path_size, path) == path->vertices.size() - 1, "Invalid mgp_path"); - // Check that the both the last vertex on path and dst_vertex are endpoints of - // the given edge. - auto *src_vertex = &path->vertices.back(); - mgp_vertex *dst_vertex{nullptr}; - if (edge->to == *src_vertex) { - dst_vertex = &edge->from; - } else if (edge->from == *src_vertex) { - dst_vertex = &edge->to; - } else { - // edge is not a continuation on src_vertex - throw std::logic_error{"The current last vertex in the path is not part of the given edge."}; - } - // Try appending edge and dst_vertex to path, preserving the original mgp_path - // instance if anything fails. - memgraph::utils::OnScopeExit scope_guard( - [path] { MG_ASSERT(Call(mgp_path_size, path) == path->vertices.size() - 1); }); - - path->edges.push_back(*edge); - path->vertices.push_back(*dst_vertex); - }); -} - -namespace { -size_t MgpPathSize(const mgp_path &path) noexcept { return path.edges.size(); } -} // namespace - -mgp_error mgp_path_size(mgp_path *path, size_t *result) { - *result = MgpPathSize(*path); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_path_vertex_at(mgp_path *path, size_t i, mgp_vertex **result) { - return WrapExceptions( - [path, i] { - const auto path_size = Call(mgp_path_size, path); - MG_ASSERT(path_size == path->vertices.size() - 1); - if (i > path_size) { - throw std::out_of_range("Vertex cannot be retrieved, because index exceeds path's size!"); - } - return &path->vertices[i]; - }, - result); -} - -mgp_error mgp_path_edge_at(mgp_path *path, size_t i, mgp_edge **result) { - return WrapExceptions( - [path, i] { - const auto path_size = Call(mgp_path_size, path); - MG_ASSERT(path_size == path->vertices.size() - 1); - if (i > path_size) { - throw std::out_of_range("Edge cannot be retrieved, because index exceeds path's size!"); - } - return &path->edges[i]; - }, - result); -} - -mgp_error mgp_path_equal(mgp_path *p1, mgp_path *p2, int *result) { - return WrapExceptions( - [p1, p2] { - const auto p1_size = MgpPathSize(*p1); - const auto p2_size = MgpPathSize(*p2); - MG_ASSERT(p1_size == p1->vertices.size() - 1); - MG_ASSERT(p2_size == p2->vertices.size() - 1); - if (p1_size != p2_size) { - return 0; - } - const auto *start1 = Call(mgp_path_vertex_at, p1, 0); - const auto *start2 = Call(mgp_path_vertex_at, p2, 0); - static_assert(noexcept(start1->impl == start2->impl)); - if (*start1 != *start2) { - return 0; - } - for (size_t i = 0; i < p1_size; ++i) { - const auto *e1 = Call(mgp_path_edge_at, p1, i); - const auto *e2 = Call(mgp_path_edge_at, p2, i); - if (*e1 != *e2) { - return 0; - } - } - return 1; - }, - result); -} - -mgp_error mgp_date_from_string(const char *string, mgp_memory *memory, mgp_date **date) { - return WrapExceptions([string, memory] { return NewRawMgpObject(memory, string); }, date); -} - -mgp_error mgp_date_from_parameters(mgp_date_parameters *parameters, mgp_memory *memory, mgp_date **date) { - return WrapExceptions([parameters, memory] { return NewRawMgpObject(memory, parameters); }, date); -} - -mgp_error mgp_date_copy(mgp_date *date, mgp_memory *memory, mgp_date **result) { - return WrapExceptions([date, memory] { return NewRawMgpObject(memory, *date); }, result); -} - -void mgp_date_destroy(mgp_date *date) { DeleteRawMgpObject(date); } - -mgp_error mgp_date_equal(mgp_date *first, mgp_date *second, int *result) { - return WrapExceptions([first, second] { return first->date == second->date; }, result); -} - -mgp_error mgp_date_get_year(mgp_date *date, int *year) { - return WrapExceptions([date] { return date->date.year; }, year); -} - -mgp_error mgp_date_get_month(mgp_date *date, int *month) { - return WrapExceptions([date] { return date->date.month; }, month); -} - -mgp_error mgp_date_get_day(mgp_date *date, int *day) { - return WrapExceptions([date] { return date->date.day; }, day); -} - -mgp_error mgp_date_timestamp(mgp_date *date, int64_t *timestamp) { - return WrapExceptions([date] { return date->date.MicrosecondsSinceEpoch(); }, timestamp); -} - -mgp_error mgp_date_now(mgp_memory *memory, mgp_date **date) { - return WrapExceptions([memory] { return NewRawMgpObject(memory, memgraph::utils::CurrentDate()); }, date); -} - -mgp_error mgp_date_add_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory, mgp_date **result) { - return WrapExceptions([date, dur, memory] { return NewRawMgpObject(memory, date->date + dur->duration); }, - result); -} - -mgp_error mgp_date_sub_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory, mgp_date **result) { - return WrapExceptions([date, dur, memory] { return NewRawMgpObject(memory, date->date - dur->duration); }, - result); -} - -mgp_error mgp_date_diff(mgp_date *first, mgp_date *second, mgp_memory *memory, mgp_duration **result) { - return WrapExceptions( - [first, second, memory] { return NewRawMgpObject(memory, first->date - second->date); }, result); -} - -mgp_error mgp_local_time_from_string(const char *string, mgp_memory *memory, mgp_local_time **local_time) { - return WrapExceptions([string, memory] { return NewRawMgpObject(memory, string); }, local_time); -} - -mgp_error mgp_local_time_from_parameters(mgp_local_time_parameters *parameters, mgp_memory *memory, - mgp_local_time **local_time) { - return WrapExceptions([parameters, memory] { return NewRawMgpObject(memory, parameters); }, - local_time); -} - -mgp_error mgp_local_time_copy(mgp_local_time *local_time, mgp_memory *memory, mgp_local_time **result) { - return WrapExceptions([local_time, memory] { return NewRawMgpObject(memory, *local_time); }, result); -} - -void mgp_local_time_destroy(mgp_local_time *local_time) { DeleteRawMgpObject(local_time); } - -mgp_error mgp_local_time_equal(mgp_local_time *first, mgp_local_time *second, int *result) { - return WrapExceptions([first, second] { return first->local_time == second->local_time; }, result); -} - -mgp_error mgp_local_time_get_hour(mgp_local_time *local_time, int *hour) { - return WrapExceptions([local_time] { return local_time->local_time.hour; }, hour); -} - -mgp_error mgp_local_time_get_minute(mgp_local_time *local_time, int *minute) { - return WrapExceptions([local_time] { return local_time->local_time.minute; }, minute); -} - -mgp_error mgp_local_time_get_second(mgp_local_time *local_time, int *second) { - return WrapExceptions([local_time] { return local_time->local_time.second; }, second); -} - -mgp_error mgp_local_time_get_millisecond(mgp_local_time *local_time, int *millisecond) { - return WrapExceptions([local_time] { return local_time->local_time.millisecond; }, millisecond); -} - -mgp_error mgp_local_time_get_microsecond(mgp_local_time *local_time, int *microsecond) { - return WrapExceptions([local_time] { return local_time->local_time.microsecond; }, microsecond); -} - -mgp_error mgp_local_time_timestamp(mgp_local_time *local_time, int64_t *timestamp) { - return WrapExceptions([local_time] { return local_time->local_time.MicrosecondsSinceEpoch(); }, timestamp); -} - -mgp_error mgp_local_time_now(mgp_memory *memory, mgp_local_time **local_time) { - return WrapExceptions( - [memory] { return NewRawMgpObject(memory, memgraph::utils::CurrentLocalTime()); }, local_time); -} - -mgp_error mgp_local_time_add_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory, - mgp_local_time **result) { - return WrapExceptions( - [local_time, dur, memory] { - return NewRawMgpObject(memory, local_time->local_time + dur->duration); - }, - result); -} - -mgp_error mgp_local_time_sub_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory, - mgp_local_time **result) { - return WrapExceptions( - [local_time, dur, memory] { - return NewRawMgpObject(memory, local_time->local_time - dur->duration); - }, - result); -} - -mgp_error mgp_local_time_diff(mgp_local_time *first, mgp_local_time *second, mgp_memory *memory, - mgp_duration **result) { - return WrapExceptions( - [first, second, memory] { return NewRawMgpObject(memory, first->local_time - second->local_time); }, - result); -} - -mgp_error mgp_local_date_time_from_string(const char *string, mgp_memory *memory, - mgp_local_date_time **local_date_time) { - return WrapExceptions([string, memory] { return NewRawMgpObject(memory, string); }, - local_date_time); -} - -mgp_error mgp_local_date_time_from_parameters(mgp_local_date_time_parameters *parameters, mgp_memory *memory, - mgp_local_date_time **local_date_time) { - return WrapExceptions([parameters, memory] { return NewRawMgpObject(memory, parameters); }, - local_date_time); -} - -mgp_error mgp_local_date_time_copy(mgp_local_date_time *local_date_time, mgp_memory *memory, - mgp_local_date_time **result) { - return WrapExceptions( - [local_date_time, memory] { return NewRawMgpObject(memory, *local_date_time); }, result); -} - -void mgp_local_date_time_destroy(mgp_local_date_time *local_date_time) { DeleteRawMgpObject(local_date_time); } - -mgp_error mgp_local_date_time_equal(mgp_local_date_time *first, mgp_local_date_time *second, int *result) { - return WrapExceptions([first, second] { return first->local_date_time == second->local_date_time; }, result); -} - -mgp_error mgp_local_date_time_get_year(mgp_local_date_time *local_date_time, int *year) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.date.year; }, year); -} - -mgp_error mgp_local_date_time_get_month(mgp_local_date_time *local_date_time, int *month) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.date.month; }, month); -} - -mgp_error mgp_local_date_time_get_day(mgp_local_date_time *local_date_time, int *day) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.date.day; }, day); -} - -mgp_error mgp_local_date_time_get_hour(mgp_local_date_time *local_date_time, int *hour) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.local_time.hour; }, hour); -} - -mgp_error mgp_local_date_time_get_minute(mgp_local_date_time *local_date_time, int *minute) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.local_time.minute; }, minute); -} - -mgp_error mgp_local_date_time_get_second(mgp_local_date_time *local_date_time, int *second) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.local_time.second; }, second); -} - -mgp_error mgp_local_date_time_get_millisecond(mgp_local_date_time *local_date_time, int *millisecond) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.local_time.millisecond; }, - millisecond); -} - -mgp_error mgp_local_date_time_get_microsecond(mgp_local_date_time *local_date_time, int *microsecond) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.local_time.microsecond; }, - microsecond); -} - -mgp_error mgp_local_date_time_timestamp(mgp_local_date_time *local_date_time, int64_t *timestamp) { - return WrapExceptions([local_date_time] { return local_date_time->local_date_time.MicrosecondsSinceEpoch(); }, - timestamp); -} - -mgp_error mgp_local_date_time_now(mgp_memory *memory, mgp_local_date_time **local_date_time) { - return WrapExceptions( - [memory] { return NewRawMgpObject(memory, memgraph::utils::CurrentLocalDateTime()); }, - local_date_time); -} - -mgp_error mgp_local_date_time_add_duration(mgp_local_date_time *local_date_time, mgp_duration *dur, mgp_memory *memory, - mgp_local_date_time **result) { - return WrapExceptions( - [local_date_time, dur, memory] { - return NewRawMgpObject(memory, local_date_time->local_date_time + dur->duration); - }, - result); -} - -mgp_error mgp_local_date_time_sub_duration(mgp_local_date_time *local_date_time, mgp_duration *dur, mgp_memory *memory, - mgp_local_date_time **result) { - return WrapExceptions( - [local_date_time, dur, memory] { - return NewRawMgpObject(memory, local_date_time->local_date_time - dur->duration); - }, - result); -} - -mgp_error mgp_local_date_time_diff(mgp_local_date_time *first, mgp_local_date_time *second, mgp_memory *memory, - mgp_duration **result) { - return WrapExceptions( - [first, second, memory] { - return NewRawMgpObject(memory, first->local_date_time - second->local_date_time); - }, - result); -} - -mgp_error mgp_duration_from_string(const char *string, mgp_memory *memory, mgp_duration **duration) { - return WrapExceptions([memory, string] { return NewRawMgpObject(memory, string); }, duration); -} - -mgp_error mgp_duration_from_parameters(mgp_duration_parameters *parameters, mgp_memory *memory, - mgp_duration **duration) { - return WrapExceptions([memory, parameters] { return NewRawMgpObject(memory, parameters); }, duration); -} - -mgp_error mgp_duration_from_microseconds(int64_t microseconds, mgp_memory *memory, mgp_duration **duration) { - return WrapExceptions([microseconds, memory] { return NewRawMgpObject(memory, microseconds); }, - duration); -} - -mgp_error mgp_duration_copy(mgp_duration *duration, mgp_memory *memory, mgp_duration **result) { - return WrapExceptions([duration, memory] { return NewRawMgpObject(memory, *duration); }, result); -} - -void mgp_duration_destroy(mgp_duration *duration) { DeleteRawMgpObject(duration); } - -mgp_error mgp_duration_get_microseconds(mgp_duration *duration, int64_t *microseconds) { - return WrapExceptions([duration] { return duration->duration.microseconds; }, microseconds); -} - -mgp_error mgp_duration_equal(mgp_duration *first, mgp_duration *second, int *result) { - return WrapExceptions([first, second] { return first->duration == second->duration; }, result); -} - -mgp_error mgp_duration_neg(mgp_duration *dur, mgp_memory *memory, mgp_duration **result) { - return WrapExceptions([memory, dur] { return NewRawMgpObject(memory, -dur->duration); }, result); -} - -mgp_error mgp_duration_add(mgp_duration *first, mgp_duration *second, mgp_memory *memory, mgp_duration **result) { - return WrapExceptions( - [memory, first, second] { return NewRawMgpObject(memory, first->duration + second->duration); }, - result); -} - -mgp_error mgp_duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory *memory, mgp_duration **result) { - return WrapExceptions( - [memory, first, second] { return NewRawMgpObject(memory, first->duration - second->duration); }, - result); -} - -/// Plugin Result - -mgp_error mgp_result_set_error_msg(mgp_result *res, const char *msg) { - return WrapExceptions([=] { - auto *memory = res->rows.get_allocator().GetMemoryResource(); - res->error_msg.emplace(msg, memory); - }); -} - -mgp_error mgp_result_new_record(mgp_result *res, mgp_result_record **result) { - return WrapExceptions( - [res] { - auto *memory = res->rows.get_allocator().GetMemoryResource(); - MG_ASSERT(res->signature, "Expected to have a valid signature"); - res->rows.push_back(mgp_result_record{ - res->signature, - memgraph::utils::pmr::map(memory)}); - return &res->rows.back(); - }, - result); -} - -mgp_error mgp_result_record_insert(mgp_result_record *record, const char *field_name, mgp_value *val) { - return WrapExceptions([=] { - auto *memory = record->values.get_allocator().GetMemoryResource(); - // Validate field_name & val satisfy the procedure's result signature. - MG_ASSERT(record->signature, "Expected to have a valid signature"); - auto find_it = record->signature->find(field_name); - if (find_it == record->signature->end()) { - throw std::out_of_range{fmt::format("The result doesn't have any field named '{}'.", field_name)}; - } - const auto *type = find_it->second.first; - if (!type->SatisfiesType(*val)) { - throw std::logic_error{ - fmt::format("The type of value doesn't satisfies the type '{}'!", type->GetPresentableName())}; - } - record->values.emplace(field_name, ToTypedValue(*val, memory)); - }); -} - -mgp_error mgp_func_result_set_error_msg(mgp_func_result *res, const char *msg, mgp_memory *memory) { - return WrapExceptions([=] { res->error_msg.emplace(msg, memory->impl); }); -} - -mgp_error mgp_func_result_set_value(mgp_func_result *res, mgp_value *value, mgp_memory *memory) { - return WrapExceptions([=] { res->value = ToTypedValue(*value, memory->impl); }); -} - -/// Graph Constructs - -void mgp_properties_iterator_destroy(mgp_properties_iterator *it) { DeleteRawMgpObject(it); } - -mgp_error mgp_properties_iterator_get(mgp_properties_iterator *it, mgp_property **result) { - return WrapExceptions( - [it]() -> mgp_property * { - if (it->current) { - return &it->property; - }; - return nullptr; - }, - result); -} - -mgp_error mgp_properties_iterator_next(mgp_properties_iterator *it, mgp_property **result) { - // Incrementing the iterator either for on-disk or in-memory - // storage, so perhaps the underlying thing can throw. - // Both copying TypedValue and/or string from PropertyName may fail to - // allocate. Also, dereferencing `it->current_it` could also throw, so - // either way return nullptr and leave `it` in undefined state. - // Hopefully iterator comparison doesn't throw, but wrap the whole thing in - // try ... catch just to be sure. - return WrapExceptions( - [it]() -> mgp_property * { - if (it->current_it == it->pvs.end()) { - MG_ASSERT(!it->current, - "Iteration is already done, so it->current should " - "have been set to std::nullopt"); - return nullptr; - } - if (++it->current_it == it->pvs.end()) { - it->current = std::nullopt; - return nullptr; - } - memgraph::utils::OnScopeExit clean_up([it] { it->current = std::nullopt; }); - it->current.emplace(memgraph::utils::pmr::string(it->graph->impl->PropertyToName(it->current_it->first), - it->GetMemoryResource()), - mgp_value(it->current_it->second, it->GetMemoryResource())); - it->property.name = it->current->first.c_str(); - it->property.value = &it->current->second; - clean_up.Disable(); - return &it->property; - }, - result); -} - -// TODO(jbajic) Fix Remove Gid -mgp_error mgp_vertex_get_id(mgp_vertex *v, mgp_vertex_id *result) { - return WrapExceptions([] { return mgp_vertex_id{.as_int = 0}; }, result); -} - -mgp_error mgp_vertex_underlying_graph_is_mutable(mgp_vertex *v, int *result) { - return mgp_graph_is_mutable(v->graph, result); -} - -namespace { -memgraph::storage::v3::PropertyValue ToPropertyValue(const mgp_value &value); - -memgraph::storage::v3::PropertyValue ToPropertyValue(const mgp_list &list) { - memgraph::storage::v3::PropertyValue result{std::vector{}}; - auto &result_list = result.ValueList(); - for (const auto &value : list.elems) { - result_list.push_back(ToPropertyValue(value)); - } - return result; -} - -memgraph::storage::v3::PropertyValue ToPropertyValue(const mgp_map &map) { - memgraph::storage::v3::PropertyValue result{std::map{}}; - auto &result_map = result.ValueMap(); - for (const auto &[key, value] : map.items) { - result_map.insert_or_assign(std::string{key}, ToPropertyValue(value)); - } - return result; -} - -memgraph::storage::v3::PropertyValue ToPropertyValue(const mgp_value &value) { - switch (value.type) { - case MGP_VALUE_TYPE_NULL: - return memgraph::storage::v3::PropertyValue{}; - case MGP_VALUE_TYPE_BOOL: - return memgraph::storage::v3::PropertyValue{value.bool_v}; - case MGP_VALUE_TYPE_INT: - return memgraph::storage::v3::PropertyValue{value.int_v}; - case MGP_VALUE_TYPE_DOUBLE: - return memgraph::storage::v3::PropertyValue{value.double_v}; - case MGP_VALUE_TYPE_STRING: - return memgraph::storage::v3::PropertyValue{std::string{value.string_v}}; - case MGP_VALUE_TYPE_LIST: - return ToPropertyValue(*value.list_v); - case MGP_VALUE_TYPE_MAP: - return ToPropertyValue(*value.map_v); - case MGP_VALUE_TYPE_DATE: - return memgraph::storage::v3::PropertyValue{memgraph::storage::v3::TemporalData{ - memgraph::storage::v3::TemporalType::Date, value.date_v->date.MicrosecondsSinceEpoch()}}; - case MGP_VALUE_TYPE_LOCAL_TIME: - return memgraph::storage::v3::PropertyValue{memgraph::storage::v3::TemporalData{ - memgraph::storage::v3::TemporalType::LocalTime, value.local_time_v->local_time.MicrosecondsSinceEpoch()}}; - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: - return memgraph::storage::v3::PropertyValue{ - memgraph::storage::v3::TemporalData{memgraph::storage::v3::TemporalType::LocalDateTime, - value.local_date_time_v->local_date_time.MicrosecondsSinceEpoch()}}; - case MGP_VALUE_TYPE_DURATION: - return memgraph::storage::v3::PropertyValue{memgraph::storage::v3::TemporalData{ - memgraph::storage::v3::TemporalType::Duration, value.duration_v->duration.microseconds}}; - case MGP_VALUE_TYPE_VERTEX: - throw ValueConversionException{"A vertex is not a valid property value! "}; - case MGP_VALUE_TYPE_EDGE: - throw ValueConversionException{"An edge is not a valid property value!"}; - case MGP_VALUE_TYPE_PATH: - throw ValueConversionException{"A path is not a valid property value!"}; - } -} -} // namespace - -mgp_error mgp_vertex_set_property(struct mgp_vertex *v, const char *property_name, mgp_value *property_value) { - return WrapExceptions([=] { - if (!MgpVertexIsMutable(*v)) { - throw ImmutableObjectException{"Cannot set a property on an immutable vertex!"}; - } - const auto prop_key = v->graph->impl->NameToProperty(property_name); - const auto result = v->impl.SetProperty(prop_key, ToPropertyValue(*property_value)); - if (result.HasError()) { - // TODO(jbajic) Fix query modules - // switch (result.GetError()) { - // case memgraph::storage::v3::Error::DELETED_OBJECT: - // throw DeletedObjectException{"Cannot set the properties of a deleted vertex!"}; - // case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - // LOG_FATAL("Query modules shouldn't have access to nonexistent objects when setting a property of a - // vertex!"); - // case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - // case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - // LOG_FATAL("Unexpected error when setting a property of a vertex."); - // case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - // throw SerializationException{"Cannot serialize setting a property of a vertex."}; - // } - } - - auto &ctx = v->graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::UPDATED_PROPERTIES] += 1; - - auto *trigger_ctx_collector = ctx->trigger_context_collector; - if (!trigger_ctx_collector || - !trigger_ctx_collector->ShouldRegisterObjectPropertyChange()) { - return; - } - using memgraph::query::v2::TypedValue; - const auto old_value = memgraph::storage::v3::PropertyToTypedValue(*result); - if (property_value->type == mgp_value_type::MGP_VALUE_TYPE_NULL) { - trigger_ctx_collector->RegisterRemovedObjectProperty(v->impl, prop_key, old_value); - return; - } - const auto new_value = ToTypedValue(*property_value, property_value->memory); - trigger_ctx_collector->RegisterSetObjectProperty(v->impl, prop_key, old_value, new_value); - }); -} - -mgp_error mgp_vertex_add_label(struct mgp_vertex *v, mgp_label label) { - return WrapExceptions([=] { - if (!MgpVertexIsMutable(*v)) { - throw ImmutableObjectException{"Cannot add a label to an immutable vertex!"}; - } - const auto label_id = v->graph->impl->NameToLabel(label.name); - const auto result = v->impl.AddLabel(label_id); - - if (result.HasError()) { - // TODO(jbajic) Fix query modules - // switch (result.GetError()) { - // case memgraph::storage::v3::Error::DELETED_OBJECT: - // throw DeletedObjectException{"Cannot add a label to a deleted vertex!"}; - // case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - // LOG_FATAL("Query modules shouldn't have access to nonexistent objects when adding a label to a vertex!"); - // case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - // case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - // LOG_FATAL("Unexpected error when adding a label to a vertex."); - // case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - // throw SerializationException{"Cannot serialize adding a label to a vertex."}; - // } - } - - auto &ctx = v->graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::CREATED_LABELS] += 1; - - if (ctx->trigger_context_collector) { - ctx->trigger_context_collector->RegisterSetVertexLabel(v->impl, label_id); - } - }); -} - -mgp_error mgp_vertex_remove_label(struct mgp_vertex *v, mgp_label label) { - return WrapExceptions([=] { - if (!MgpVertexIsMutable(*v)) { - throw ImmutableObjectException{"Cannot remove a label from an immutable vertex!"}; - } - const auto label_id = v->graph->impl->NameToLabel(label.name); - const auto result = v->impl.RemoveLabel(label_id); - - if (result.HasError()) { - // TODO(jbajic) Fix query modules - // switch (result.GetError()) { - // case memgraph::storage::v3::Error::DELETED_OBJECT: - // throw DeletedObjectException{"Cannot remove a label from a deleted vertex!"}; - // case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - // LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing a label from a - // vertex!"); - // case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - // case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - // LOG_FATAL("Unexpected error when removing a label from a vertex."); - // case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - // throw SerializationException{"Cannot serialize removing a label from a vertex."}; - // } - } - - auto &ctx = v->graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::DELETED_LABELS] += 1; - - if (ctx->trigger_context_collector) { - ctx->trigger_context_collector->RegisterRemovedVertexLabel(v->impl, label_id); - } - }); -} - -mgp_error mgp_vertex_copy(mgp_vertex *v, mgp_memory *memory, mgp_vertex **result) { - return WrapExceptions([v, memory] { return NewRawMgpObject(memory, *v); }, result); -} - -void mgp_vertex_destroy(mgp_vertex *v) { DeleteRawMgpObject(v); } - -mgp_error mgp_vertex_equal(mgp_vertex *v1, mgp_vertex *v2, int *result) { - // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) - static_assert(noexcept(*result = *v1 == *v2 ? 1 : 0)); - *result = *v1 == *v2 ? 1 : 0; - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_vertex_labels_count(mgp_vertex *v, size_t *result) { - return WrapExceptions( - [v]() -> size_t { - auto maybe_labels = v->impl.Labels(v->graph->view); - if (maybe_labels.HasError()) { - switch (maybe_labels.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get the labels of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when getting vertex labels!"); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting vertex labels."); - } - } - return maybe_labels->size(); - }, - result); -} - -mgp_error mgp_vertex_label_at(mgp_vertex *v, size_t i, mgp_label *result) { - return WrapExceptions( - [v, i]() -> const char * { - // TODO: Maybe it's worth caching this in mgp_vertex. - auto maybe_labels = v->impl.Labels(v->graph->view); - if (maybe_labels.HasError()) { - switch (maybe_labels.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get a label of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when getting a label of a vertex!"); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting a label of a vertex."); - } - } - if (i >= maybe_labels->size()) { - throw std::out_of_range("Label cannot be retrieved, because index exceeds the number of labels!"); - } - const auto &label = (*maybe_labels)[i]; - static_assert(std::is_lvalue_reference_vgraph->impl->LabelToName(label))>, - "Expected LabelToName to return a pointer or reference, so we " - "don't have to take a copy and manage memory."); - const auto &name = v->graph->impl->LabelToName(label); - return name.c_str(); - }, - &result->name); -} - -mgp_error mgp_vertex_has_label_named(mgp_vertex *v, const char *name, int *result) { - return WrapExceptions( - [v, name] { - memgraph::storage::v3::LabelId label; - label = v->graph->impl->NameToLabel(name); - - auto maybe_has_label = v->impl.HasLabel(v->graph->view, label); - if (maybe_has_label.HasError()) { - switch (maybe_has_label.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot check the existence of a label on a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when checking the existence of a label " - "on " - "a vertex!"); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when checking the existence of a label on a vertex."); - } - } - return *maybe_has_label ? 1 : 0; - }, - result); -} - -mgp_error mgp_vertex_has_label(mgp_vertex *v, mgp_label label, int *result) { - return mgp_vertex_has_label_named(v, label.name, result); -} - -mgp_error mgp_vertex_get_property(mgp_vertex *v, const char *name, mgp_memory *memory, mgp_value **result) { - return WrapExceptions( - [v, name, memory]() -> mgp_value * { - const auto &key = v->graph->impl->NameToProperty(name); - auto maybe_prop = v->impl.GetProperty(v->graph->view, key); - if (maybe_prop.HasError()) { - switch (maybe_prop.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get a property of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting a property of a vertex."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting a property of a vertex."); - } - } - return NewRawMgpObject(memory, std::move(*maybe_prop)); - }, - result); -} - -mgp_error mgp_vertex_iter_properties(mgp_vertex *v, mgp_memory *memory, mgp_properties_iterator **result) { - // NOTE: This copies the whole properties into the iterator. - // TODO: Think of a good way to avoid the copy which doesn't just rely on some - // assumption that storage may return a pointer to the property store. This - // will probably require a different API in storage. - return WrapExceptions( - [v, memory] { - auto maybe_props = v->impl.Properties(v->graph->view); - if (maybe_props.HasError()) { - switch (maybe_props.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get the properties of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting the properties of a " - "vertex."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting the properties of a vertex."); - } - } - return NewRawMgpObject(memory, v->graph, std::move(*maybe_props)); - }, - result); -} - -void mgp_edges_iterator_destroy(mgp_edges_iterator *it) { DeleteRawMgpObject(it); } - -mgp_error mgp_vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory, mgp_edges_iterator **result) { - return WrapExceptions( - [v, memory] { - auto it = NewMgpObject(memory, *v); - MG_ASSERT(it != nullptr); - - auto maybe_edges = v->impl.InEdges(v->graph->view); - if (maybe_edges.HasError()) { - switch (maybe_edges.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get the inbound edges of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting the inbound edges of a " - "vertex."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting the inbound edges of a vertex."); - } - } - it->in.emplace(std::move(*maybe_edges)); - it->in_it.emplace(it->in->begin()); - if (*it->in_it != it->in->end()) { - it->current_e.emplace(**it->in_it, v->graph, it->GetMemoryResource()); - } - - return it.release(); - }, - result); -} - -mgp_error mgp_vertex_iter_out_edges(mgp_vertex *v, mgp_memory *memory, mgp_edges_iterator **result) { - return WrapExceptions( - [v, memory] { - auto it = NewMgpObject(memory, *v); - MG_ASSERT(it != nullptr); - - auto maybe_edges = v->impl.OutEdges(v->graph->view); - if (maybe_edges.HasError()) { - switch (maybe_edges.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get the outbound edges of a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting the outbound edges of a " - "vertex."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting the outbound edges of a vertex."); - } - } - it->out.emplace(std::move(*maybe_edges)); - it->out_it.emplace(it->out->begin()); - if (*it->out_it != it->out->end()) { - it->current_e.emplace(**it->out_it, v->graph, it->GetMemoryResource()); - } - - return it.release(); - }, - result); -} - -mgp_error mgp_edges_iterator_underlying_graph_is_mutable(mgp_edges_iterator *it, int *result) { - return mgp_vertex_underlying_graph_is_mutable(&it->source_vertex, result); -} - -mgp_error mgp_edges_iterator_get(mgp_edges_iterator *it, mgp_edge **result) { - return WrapExceptions( - [it]() -> mgp_edge * { - if (it->current_e.has_value()) { - return &*it->current_e; - } - return nullptr; - }, - result); -} - -mgp_error mgp_edges_iterator_next(mgp_edges_iterator *it, mgp_edge **result) { - return WrapExceptions( - [it] { - MG_ASSERT(it->in || it->out); - auto next = [&](auto *impl_it, const auto &end) -> mgp_edge * { - if (*impl_it == end) { - MG_ASSERT(!it->current_e, - "Iteration is already done, so it->current_e " - "should have been set to std::nullopt"); - return nullptr; - } - if (++(*impl_it) == end) { - it->current_e = std::nullopt; - return nullptr; - } - it->current_e.emplace(**impl_it, it->source_vertex.graph, it->GetMemoryResource()); - return &*it->current_e; - }; - if (it->in_it) { - return next(&*it->in_it, it->in->end()); - } - return next(&*it->out_it, it->out->end()); - }, - result); -} - -mgp_error mgp_edge_get_id(mgp_edge *e, mgp_edge_id *result) { - return WrapExceptions([e] { return mgp_edge_id{.as_int = e->impl.Gid().AsInt()}; }, result); -} - -mgp_error mgp_edge_underlying_graph_is_mutable(mgp_edge *e, int *result) { - return mgp_vertex_underlying_graph_is_mutable(&e->from, result); -} - -mgp_error mgp_edge_copy(mgp_edge *e, mgp_memory *memory, mgp_edge **result) { - return WrapExceptions([e, memory] { return mgp_edge::Copy(*e, *memory); }, result); -} - -void mgp_edge_destroy(mgp_edge *e) { DeleteRawMgpObject(e); } - -mgp_error mgp_edge_equal(mgp_edge *e1, mgp_edge *e2, int *result) { - // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) - static_assert(noexcept(*result = *e1 == *e2 ? 1 : 0)); - *result = *e1 == *e2 ? 1 : 0; - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_edge_get_type(mgp_edge *e, mgp_edge_type *result) { - return WrapExceptions( - [e] { - const auto &name = e->from.graph->impl->EdgeTypeToName(e->impl.EdgeType()); - static_assert(std::is_lvalue_reference_vfrom.graph->impl->EdgeTypeToName(e->impl.EdgeType()))>, - "Expected EdgeTypeToName to return a pointer or reference, so we " - "don't have to take a copy and manage memory."); - return name.c_str(); - }, - &result->name); -} - -mgp_error mgp_edge_get_from(mgp_edge *e, mgp_vertex **result) { - *result = &e->from; - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_edge_get_to(mgp_edge *e, mgp_vertex **result) { - *result = &e->to; - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_edge_get_property(mgp_edge *e, const char *name, mgp_memory *memory, mgp_value **result) { - return WrapExceptions( - [e, name, memory] { - const auto &key = e->from.graph->impl->NameToProperty(name); - auto view = e->from.graph->view; - auto maybe_prop = e->impl.GetProperty(view, key); - if (maybe_prop.HasError()) { - switch (maybe_prop.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get a property of a deleted edge!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting a property of an edge."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting a property of an edge."); - } - } - return NewRawMgpObject(memory, std::move(*maybe_prop)); - }, - result); -} - -mgp_error mgp_edge_set_property(struct mgp_edge *e, const char *property_name, mgp_value *property_value) { - return WrapExceptions([=] { - if (!MgpEdgeIsMutable(*e)) { - throw ImmutableObjectException{"Cannot set a property on an immutable edge!"}; - } - const auto prop_key = e->from.graph->impl->NameToProperty(property_name); - const auto result = e->impl.SetProperty(prop_key, ToPropertyValue(*property_value)); - - if (result.HasError()) { - switch (result.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot set the properties of a deleted edge!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when setting a property of an edge!"); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - throw std::logic_error{"Cannot set the properties of edges, because properties on edges are disabled!"}; - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - LOG_FATAL("Unexpected error when setting a property of an edge."); - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - throw SerializationException{"Cannot serialize setting a property of an edge."}; - } - } - - auto &ctx = e->from.graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::UPDATED_PROPERTIES] += 1; - - auto *trigger_ctx_collector = e->from.graph->ctx->trigger_context_collector; - if (!trigger_ctx_collector || - !trigger_ctx_collector->ShouldRegisterObjectPropertyChange()) { - return; - } - using memgraph::query::v2::TypedValue; - const auto old_value = memgraph::storage::v3::PropertyToTypedValue(*result); - if (property_value->type == mgp_value_type::MGP_VALUE_TYPE_NULL) { - e->from.graph->ctx->trigger_context_collector->RegisterRemovedObjectProperty(e->impl, prop_key, old_value); - return; - } - const auto new_value = ToTypedValue(*property_value, property_value->memory); - e->from.graph->ctx->trigger_context_collector->RegisterSetObjectProperty(e->impl, prop_key, old_value, new_value); - }); -} - -mgp_error mgp_edge_iter_properties(mgp_edge *e, mgp_memory *memory, mgp_properties_iterator **result) { - // NOTE: This copies the whole properties into iterator. - // TODO: Think of a good way to avoid the copy which doesn't just rely on some - // assumption that storage may return a pointer to the property store. This - // will probably require a different API in storage. - return WrapExceptions( - [e, memory] { - auto view = e->from.graph->view; - auto maybe_props = e->impl.Properties(view); - if (maybe_props.HasError()) { - switch (maybe_props.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot get the properties of a deleted edge!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL( - "Query modules shouldn't have access to nonexistent objects when getting the properties of an edge."); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - LOG_FATAL("Unexpected error when getting the properties of an edge."); - } - } - return NewRawMgpObject(memory, e->from.graph, std::move(*maybe_props)); - }, - result); -} - -mgp_error mgp_graph_get_vertex_by_id(mgp_graph *graph, mgp_vertex_id id, mgp_memory *memory, mgp_vertex **result) { - return WrapExceptions( - []() -> mgp_vertex * { - // TODO(jbajic) Fix Remove Gid - // auto maybe_vertex = graph->impl->FindVertex(0); - // if (maybe_vertex) { - // return NewRawMgpObject(memory, *maybe_vertex, graph); - // } - return nullptr; - }, - result); -} - -mgp_error mgp_graph_is_mutable(mgp_graph *graph, int *result) { - *result = MgpGraphIsMutable(*graph) ? 1 : 0; - return mgp_error::MGP_ERROR_NO_ERROR; -}; - -// TODO(jbajic) Fix Remove Gid -mgp_error mgp_graph_create_vertex(struct mgp_graph * /*graph*/, mgp_memory * /*memory*/, mgp_vertex ** /*result*/) { - // return WrapExceptions( - // [=] { - // if (!MgpGraphIsMutable(*graph)) { - // throw ImmutableObjectException{"Cannot create a vertex in an immutable graph!"}; - // } - // auto vertex = graph->impl->InsertVertex(); - - // auto &ctx = graph->ctx; - // ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::CREATED_NODES] += 1; - - // if (ctx->trigger_context_collector) { - // ctx->trigger_context_collector->RegisterCreatedObject(vertex); - // } - // return NewRawMgpObject(memory, nullptr, graph); - // }, - // result); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_graph_delete_vertex(struct mgp_graph *graph, mgp_vertex *vertex) { - return WrapExceptions([=] { - if (!MgpGraphIsMutable(*graph)) { - throw ImmutableObjectException{"Cannot remove a vertex from an immutable graph!"}; - } - const auto result = graph->impl->RemoveVertex(&vertex->impl); - - if (result.HasError()) { - switch (result.GetError()) { - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing a vertex!"); - case memgraph::storage::v3::Error::DELETED_OBJECT: - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - LOG_FATAL("Unexpected error when removing a vertex."); - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - throw std::logic_error{"Cannot remove a vertex that has edges!"}; - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - throw SerializationException{"Cannot serialize removing a vertex."}; - } - } - - if (!*result) { - return; - } - - auto &ctx = graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::DELETED_NODES] += 1; - - if (ctx->trigger_context_collector) { - ctx->trigger_context_collector->RegisterDeletedObject(**result); - } - }); -} - -mgp_error mgp_graph_detach_delete_vertex(struct mgp_graph *graph, mgp_vertex *vertex) { - return WrapExceptions([=] { - if (!MgpGraphIsMutable(*graph)) { - throw ImmutableObjectException{"Cannot remove a vertex from an immutable graph!"}; - } - const auto result = graph->impl->DetachRemoveVertex(&vertex->impl); - - if (result.HasError()) { - switch (result.GetError()) { - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing a vertex!"); - case memgraph::storage::v3::Error::DELETED_OBJECT: - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - LOG_FATAL("Unexpected error when removing a vertex."); - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - throw SerializationException{"Cannot serialize removing a vertex."}; - } - } - - if (!*result) { - return; - } - - auto &ctx = graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::DELETED_NODES] += 1; - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::DELETED_EDGES] += - static_cast((*result)->second.size()); - - auto *trigger_ctx_collector = ctx->trigger_context_collector; - if (!trigger_ctx_collector) { - return; - } - // TODO(jbajic) Fix Remove Gid - // trigger_ctx_collector->RegisterDeletedObject((*result)->first); - // if (!trigger_ctx_collector->ShouldRegisterDeletedObject()) { - // return; - // } - for (const auto &edge : (*result)->second) { - trigger_ctx_collector->RegisterDeletedObject(edge); - } - }); -} - -mgp_error mgp_graph_create_edge(mgp_graph *graph, mgp_vertex *from, mgp_vertex *to, mgp_edge_type type, - mgp_memory *memory, mgp_edge **result) { - return WrapExceptions( - [=] { - if (!MgpGraphIsMutable(*graph)) { - throw ImmutableObjectException{"Cannot create an edge in an immutable graph!"}; - } - - auto edge = graph->impl->InsertEdge(&from->impl, &to->impl, from->graph->impl->NameToEdgeType(type.name)); - if (edge.HasError()) { - switch (edge.GetError()) { - case memgraph::storage::v3::Error::DELETED_OBJECT: - throw DeletedObjectException{"Cannot add an edge to a deleted vertex!"}; - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when creating an edge!"); - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - LOG_FATAL("Unexpected error when creating an edge."); - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - throw SerializationException{"Cannot serialize creating an edge."}; - } - } - auto &ctx = graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::CREATED_EDGES] += 1; - - if (ctx->trigger_context_collector) { - ctx->trigger_context_collector->RegisterCreatedObject(*edge); - } - return NewRawMgpObject(memory, edge.GetValue(), from->graph); - }, - result); -} - -mgp_error mgp_graph_delete_edge(struct mgp_graph *graph, mgp_edge *edge) { - return WrapExceptions([=] { - if (!MgpGraphIsMutable(*graph)) { - throw ImmutableObjectException{"Cannot remove an edge from an immutable graph!"}; - } - const auto result = graph->impl->RemoveEdge(&edge->impl); - - if (result.HasError()) { - switch (result.GetError()) { - case memgraph::storage::v3::Error::NONEXISTENT_OBJECT: - LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing an edge!"); - case memgraph::storage::v3::Error::DELETED_OBJECT: - case memgraph::storage::v3::Error::PROPERTIES_DISABLED: - case memgraph::storage::v3::Error::VERTEX_HAS_EDGES: - LOG_FATAL("Unexpected error when removing an edge."); - case memgraph::storage::v3::Error::SERIALIZATION_ERROR: - throw SerializationException{"Cannot serialize removing an edge."}; - } - } - - if (!*result) { - return; - } - auto &ctx = graph->ctx; - - ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::DELETED_EDGES] += 1; - if (ctx->trigger_context_collector) { - ctx->trigger_context_collector->RegisterDeletedObject(**result); - } - }); -} - -void mgp_vertices_iterator_destroy(mgp_vertices_iterator *it) { DeleteRawMgpObject(it); } - -mgp_error mgp_graph_iter_vertices(mgp_graph *graph, mgp_memory *memory, mgp_vertices_iterator **result) { - return WrapExceptions([graph, memory] { return NewRawMgpObject(memory, graph); }, result); -} - -mgp_error mgp_vertices_iterator_underlying_graph_is_mutable(mgp_vertices_iterator *it, int *result) { - return mgp_graph_is_mutable(it->graph, result); -} - -mgp_error mgp_vertices_iterator_get(mgp_vertices_iterator *it, mgp_vertex **result) { - return WrapExceptions( - [it]() -> mgp_vertex * { - if (it->current_v.has_value()) { - return &*it->current_v; - } - return nullptr; - }, - result); -} - -mgp_error mgp_vertices_iterator_next(mgp_vertices_iterator *it, mgp_vertex **result) { - return WrapExceptions( - [it]() -> mgp_vertex * { - if (it->current_it == it->vertices.end()) { - MG_ASSERT(!it->current_v, - "Iteration is already done, so it->current_v " - "should have been set to std::nullopt"); - return nullptr; - } - if (++it->current_it == it->vertices.end()) { - it->current_v = std::nullopt; - return nullptr; - } - memgraph::utils::OnScopeExit clean_up([it] { it->current_v = std::nullopt; }); - it->current_v.emplace(*it->current_it, it->graph, it->GetMemoryResource()); - clean_up.Disable(); - return &*it->current_v; - }, - result); -} - -/// Type System -/// -/// All types are allocated globally, so that we simplify the API and minimize -/// allocations done for types. - -namespace { -void NoOpCypherTypeDeleter(CypherType * /*type*/) {} -} // namespace - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_MGP_TYPE_GETTER(cypher_type_name, mgp_type_name) \ - mgp_error mgp_type_##mgp_type_name(mgp_type **result) { \ - return WrapExceptions( \ - [] { \ - static cypher_type_name##Type impl; \ - static mgp_type mgp_type_name_type{CypherTypePtr(&impl, NoOpCypherTypeDeleter)}; \ - return &mgp_type_name_type; \ - }, \ - result); \ - } - -DEFINE_MGP_TYPE_GETTER(Any, any); -DEFINE_MGP_TYPE_GETTER(Bool, bool); -DEFINE_MGP_TYPE_GETTER(String, string); -DEFINE_MGP_TYPE_GETTER(Int, int); -DEFINE_MGP_TYPE_GETTER(Float, float); -DEFINE_MGP_TYPE_GETTER(Number, number); -DEFINE_MGP_TYPE_GETTER(Map, map); -DEFINE_MGP_TYPE_GETTER(Node, node); -DEFINE_MGP_TYPE_GETTER(Relationship, relationship); -DEFINE_MGP_TYPE_GETTER(Path, path); -DEFINE_MGP_TYPE_GETTER(Date, date); -DEFINE_MGP_TYPE_GETTER(LocalTime, local_time); -DEFINE_MGP_TYPE_GETTER(LocalDateTime, local_date_time); -DEFINE_MGP_TYPE_GETTER(Duration, duration); - -mgp_error mgp_type_list(mgp_type *type, mgp_type **result) { - return WrapExceptions( - [type] { - // Maps `type` to corresponding instance of ListType. - static memgraph::utils::pmr::map gListTypes(memgraph::utils::NewDeleteResource()); - static memgraph::utils::SpinLock lock; - std::lock_guard guard(lock); - auto found_it = gListTypes.find(type); - if (found_it != gListTypes.end()) { - return &found_it->second; - } - auto alloc = gListTypes.get_allocator(); - CypherTypePtr impl( - alloc.new_object( - // Just obtain the pointer to original impl, don't own it. - CypherTypePtr(type->impl.get(), NoOpCypherTypeDeleter), alloc.GetMemoryResource()), - [alloc](CypherType *base_ptr) mutable { alloc.delete_object(static_cast(base_ptr)); }); - return &gListTypes.emplace(type, mgp_type{std::move(impl)}).first->second; - }, - result); -} - -mgp_error mgp_type_nullable(mgp_type *type, mgp_type **result) { - return WrapExceptions( - [type] { - // Maps `type` to corresponding instance of NullableType. - static memgraph::utils::pmr::map gNullableTypes(memgraph::utils::NewDeleteResource()); - static memgraph::utils::SpinLock lock; - std::lock_guard guard(lock); - auto found_it = gNullableTypes.find(type); - if (found_it != gNullableTypes.end()) return &found_it->second; - - auto alloc = gNullableTypes.get_allocator(); - auto impl = - NullableType::Create(CypherTypePtr(type->impl.get(), NoOpCypherTypeDeleter), alloc.GetMemoryResource()); - return &gNullableTypes.emplace(type, mgp_type{std::move(impl)}).first->second; - }, - result); -} - -namespace { -mgp_proc *mgp_module_add_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, - const ProcedureInfo &procedure_info) { - if (!IsValidIdentifierName(name)) { - throw std::invalid_argument{fmt::format("Invalid procedure name: {}", name)}; - } - if (module->procedures.find(name) != module->procedures.end()) { - throw std::logic_error{fmt::format("Procedure already exists with name '{}'", name)}; - }; - - auto *memory = module->procedures.get_allocator().GetMemoryResource(); - // May throw std::bad_alloc, std::length_error - return &module->procedures.emplace(name, mgp_proc(name, cb, memory, procedure_info)).first->second; -} -} // namespace - -mgp_error mgp_module_add_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, mgp_proc **result) { - return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, {.is_write = false}); }, result); -} - -mgp_error mgp_module_add_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, mgp_proc **result) { - return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, {.is_write = true}); }, result); -} - -namespace { -template -concept IsCallable = memgraph::utils::SameAsAnyOf; - -template -mgp_error MgpAddArg(TCall &callable, const std::string &name, mgp_type &type) { - return WrapExceptions([&]() mutable { - static constexpr std::string_view type_name = std::invoke([]() constexpr { - if constexpr (std::is_same_v) { - return "procedure"; - } else if constexpr (std::is_same_v) { - return "function"; - } - }); - - if (!IsValidIdentifierName(name.c_str())) { - throw std::invalid_argument{fmt::format("Invalid argument name for {} '{}': {}", type_name, callable.name, name)}; - } - if (!callable.opt_args.empty()) { - throw std::logic_error{fmt::format("Cannot add required argument '{}' to {} '{}' after adding any optional one", - name, type_name, callable.name)}; - } - callable.args.emplace_back(name, type.impl.get()); - }); -} - -template -mgp_error MgpAddOptArg(TCall &callable, const std::string name, mgp_type &type, mgp_value &default_value) { - return WrapExceptions([&]() mutable { - static constexpr std::string_view type_name = std::invoke([]() constexpr { - if constexpr (std::is_same_v) { - return "procedure"; - } else if constexpr (std::is_same_v) { - return "function"; - } - }); - - if (!IsValidIdentifierName(name.c_str())) { - throw std::invalid_argument{fmt::format("Invalid argument name for {} '{}': {}", type_name, callable.name, name)}; - } - switch (MgpValueGetType(default_value)) { - case MGP_VALUE_TYPE_VERTEX: - case MGP_VALUE_TYPE_EDGE: - case MGP_VALUE_TYPE_PATH: - // default_value must not be a graph element. - throw ValueConversionException{"Default value of argument '{}' of {} '{}' name must not be a graph element!", - name, type_name, callable.name}; - case MGP_VALUE_TYPE_NULL: - case MGP_VALUE_TYPE_BOOL: - case MGP_VALUE_TYPE_INT: - case MGP_VALUE_TYPE_DOUBLE: - case MGP_VALUE_TYPE_STRING: - case MGP_VALUE_TYPE_LIST: - case MGP_VALUE_TYPE_MAP: - case MGP_VALUE_TYPE_DATE: - case MGP_VALUE_TYPE_LOCAL_TIME: - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: - case MGP_VALUE_TYPE_DURATION: - break; - } - // Default value must be of required `type`. - if (!type.impl->SatisfiesType(default_value)) { - throw std::logic_error{fmt::format("The default value of argument '{}' for {} '{}' doesn't satisfy type '{}'", - name, type_name, callable.name, type.impl->GetPresentableName())}; - } - auto *memory = callable.opt_args.get_allocator().GetMemoryResource(); - callable.opt_args.emplace_back(memgraph::utils::pmr::string(name, memory), type.impl.get(), - ToTypedValue(default_value, memory)); - }); -} -} // namespace - -mgp_error mgp_proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) { - return MgpAddArg(*proc, std::string(name), *type); -} - -mgp_error mgp_proc_add_opt_arg(mgp_proc *proc, const char *name, mgp_type *type, mgp_value *default_value) { - return MgpAddOptArg(*proc, std::string(name), *type, *default_value); -} - -mgp_error mgp_func_add_arg(mgp_func *func, const char *name, mgp_type *type) { - return MgpAddArg(*func, std::string(name), *type); -} - -mgp_error mgp_func_add_opt_arg(mgp_func *func, const char *name, mgp_type *type, mgp_value *default_value) { - return MgpAddOptArg(*func, std::string(name), *type, *default_value); -} - -namespace { - -template -concept ModuleProperties = memgraph::utils::SameAsAnyOf; - -template -mgp_error AddResultToProp(T *prop, const char *name, mgp_type *type, bool is_deprecated) noexcept { - return WrapExceptions([=] { - if (!IsValidIdentifierName(name)) { - throw std::invalid_argument{fmt::format("Invalid result name for procedure '{}': {}", prop->name, name)}; - } - if (prop->results.find(name) != prop->results.end()) { - throw std::logic_error{fmt::format("Result already exists with name '{}' for procedure '{}'", name, prop->name)}; - }; - auto *memory = prop->results.get_allocator().GetMemoryResource(); - prop->results.emplace(memgraph::utils::pmr::string(name, memory), std::make_pair(type->impl.get(), is_deprecated)); - }); -} - -} // namespace - -mgp_error mgp_proc_add_result(mgp_proc *proc, const char *name, mgp_type *type) { - return AddResultToProp(proc, name, type, false); -} - -mgp_error MgpTransAddFixedResult(mgp_trans *trans) noexcept { - if (const auto err = AddResultToProp(trans, "query", Call(mgp_type_string), false); - err != mgp_error::MGP_ERROR_NO_ERROR) { - return err; - } - return AddResultToProp(trans, "parameters", Call(mgp_type_nullable, Call(mgp_type_map)), - false); -} - -mgp_error mgp_proc_add_deprecated_result(mgp_proc *proc, const char *name, mgp_type *type) { - return AddResultToProp(proc, name, type, true); -} - -int mgp_must_abort(mgp_graph *graph) { - MG_ASSERT(graph->ctx); - static_assert(noexcept(memgraph::query::v2::MustAbort(*graph->ctx))); - return memgraph::query::v2::MustAbort(*graph->ctx) ? 1 : 0; -} - -namespace memgraph::query::v2::procedure { - -namespace { - -// Print the value in user presentable fashion. -// @throw std::bad_alloc -// @throw std::length_error -std::ostream &PrintValue(const TypedValue &value, std::ostream *stream) { - switch (value.type()) { - case TypedValue::Type::Null: - return (*stream) << "Null"; - case TypedValue::Type::Bool: - return (*stream) << (value.ValueBool() ? "true" : "false"); - case TypedValue::Type::Int: - return (*stream) << value.ValueInt(); - case TypedValue::Type::Double: - return (*stream) << value.ValueDouble(); - case TypedValue::Type::String: - // String value should be escaped, this allocates a new string. - return (*stream) << memgraph::utils::Escape(value.ValueString()); - case TypedValue::Type::List: - (*stream) << "["; - memgraph::utils::PrintIterable(*stream, value.ValueList(), ", ", - [](auto &stream, const auto &elem) { PrintValue(elem, &stream); }); - return (*stream) << "]"; - case TypedValue::Type::Map: - (*stream) << "{"; - memgraph::utils::PrintIterable(*stream, value.ValueMap(), ", ", [](auto &stream, const auto &item) { - // Map keys are not escaped strings. - stream << item.first << ": "; - PrintValue(item.second, &stream); - }); - return (*stream) << "}"; - case TypedValue::Type::Date: - return (*stream) << value.ValueDate(); - case TypedValue::Type::LocalTime: - return (*stream) << value.ValueLocalTime(); - case TypedValue::Type::LocalDateTime: - return (*stream) << value.ValueLocalDateTime(); - case TypedValue::Type::Duration: - return (*stream) << value.ValueDuration(); - case TypedValue::Type::Vertex: - case TypedValue::Type::Edge: - case TypedValue::Type::Path: - LOG_FATAL("value must not be a graph element"); - } -} - -} // namespace - -void PrintProcSignature(const mgp_proc &proc, std::ostream *stream) { - (*stream) << proc.name << "("; - memgraph::utils::PrintIterable(*stream, proc.args, ", ", [](auto &stream, const auto &arg) { - stream << arg.first << " :: " << arg.second->GetPresentableName(); - }); - if (!proc.args.empty() && !proc.opt_args.empty()) (*stream) << ", "; - memgraph::utils::PrintIterable(*stream, proc.opt_args, ", ", [](auto &stream, const auto &arg) { - stream << std::get<0>(arg) << " = "; - PrintValue(std::get<2>(arg), &stream) << " :: " << std::get<1>(arg)->GetPresentableName(); - }); - (*stream) << ") :: ("; - memgraph::utils::PrintIterable(*stream, proc.results, ", ", [](auto &stream, const auto &name_result) { - const auto &[type, is_deprecated] = name_result.second; - if (is_deprecated) stream << "DEPRECATED "; - stream << name_result.first << " :: " << type->GetPresentableName(); - }); - (*stream) << ")"; -} - -void PrintFuncSignature(const mgp_func &func, std::ostream &stream) { - stream << func.name << "("; - utils::PrintIterable(stream, func.args, ", ", [](auto &stream, const auto &arg) { - stream << arg.first << " :: " << arg.second->GetPresentableName(); - }); - if (!func.args.empty() && !func.opt_args.empty()) { - stream << ", "; - } - utils::PrintIterable(stream, func.opt_args, ", ", [](auto &stream, const auto &arg) { - const auto &[name, type, default_val] = arg; - stream << name << " = "; - PrintValue(default_val, &stream) << " :: " << type->GetPresentableName(); - }); - stream << ")"; -} - -bool IsValidIdentifierName(const char *name) { - if (!name) return false; - std::regex regex("[_[:alpha:]][_[:alnum:]]*"); - return std::regex_match(name, regex); -} - -} // namespace memgraph::query::v2::procedure - -namespace { -using StreamSourceType = memgraph::query::v2::stream::StreamSourceType; - -class InvalidMessageFunction : public std::invalid_argument { - public: - InvalidMessageFunction(const StreamSourceType type, const std::string_view function_name) - : std::invalid_argument{fmt::format("'{}' is not defined for a message from a stream of type '{}'", function_name, - StreamSourceTypeToString(type))} {} -}; - -StreamSourceType MessageToStreamSourceType(const mgp_message::KafkaMessage & /*msg*/) { - return StreamSourceType::KAFKA; -} - -StreamSourceType MessageToStreamSourceType(const mgp_message::PulsarMessage & /*msg*/) { - return StreamSourceType::PULSAR; -} - -mgp_source_type StreamSourceTypeToMgpSourceType(const StreamSourceType type) { - switch (type) { - case StreamSourceType::KAFKA: - return mgp_source_type::KAFKA; - case StreamSourceType::PULSAR: - return mgp_source_type::PULSAR; - } -} - -} // namespace - -mgp_error mgp_message_source_type(mgp_message *message, mgp_source_type *result) { - return WrapExceptions( - [message] { - return std::visit(memgraph::utils::Overloaded{[](const auto &message) { - return StreamSourceTypeToMgpSourceType(MessageToStreamSourceType(message)); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_payload(mgp_message *message, const char **result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Payload().data(); }, - [](const mgp_message::PulsarMessage &msg) { return msg.Payload().data(); }, - [](const auto &msg) -> const char * { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "payload"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_payload_size(mgp_message *message, size_t *result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Payload().size(); }, - [](const mgp_message::PulsarMessage &msg) { return msg.Payload().size(); }, - [](const auto &msg) -> size_t { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "payload_size"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_topic_name(mgp_message *message, const char **result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->TopicName().data(); }, - [](const mgp_message::PulsarMessage &msg) { return msg.TopicName().data(); }, - [](const auto &msg) -> const char * { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "topic_name"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_key(mgp_message *message, const char **result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Key().data(); }, - [](const auto &msg) -> const char * { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "key"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_key_size(mgp_message *message, size_t *result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Key().size(); }, - [](const auto &msg) -> size_t { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "key_size"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_timestamp(mgp_message *message, int64_t *result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Timestamp(); }, - [](const auto &msg) -> int64_t { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "timestamp"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_message_offset(struct mgp_message *message, int64_t *result) { - return WrapExceptions( - [message] { - return std::visit( - memgraph::utils::Overloaded{[](const mgp_message::KafkaMessage &msg) { return msg->Offset(); }, - [](const auto &msg) -> int64_t { - throw InvalidMessageFunction(MessageToStreamSourceType(msg), "offset"); - }}, - message->msg); - }, - result); -} - -mgp_error mgp_messages_size(mgp_messages *messages, size_t *result) { - static_assert(noexcept(messages->messages.size())); - *result = messages->messages.size(); - return mgp_error::MGP_ERROR_NO_ERROR; -} - -mgp_error mgp_messages_at(mgp_messages *messages, size_t index, mgp_message **result) { - return WrapExceptions( - [messages, index] { - if (index >= Call(mgp_messages_size, messages)) { - throw std::out_of_range("Message cannot be retrieved, because index exceeds messages' size!"); - } - return &messages->messages[index]; - }, - result); -} - -mgp_error mgp_module_add_transformation(mgp_module *module, const char *name, mgp_trans_cb cb) { - return WrapExceptions([=] { - if (!IsValidIdentifierName(name)) { - throw std::invalid_argument{fmt::format("Invalid transformation name: {}", name)}; - } - if (module->transformations.find(name) != module->transformations.end()) { - throw std::logic_error{fmt::format("Transformation already exists with name '{}'", name)}; - }; - auto *memory = module->transformations.get_allocator().GetMemoryResource(); - module->transformations.emplace(name, mgp_trans(name, cb, memory)); - }); -} - -mgp_error mgp_module_add_function(mgp_module *module, const char *name, mgp_func_cb cb, mgp_func **result) { - return WrapExceptions( - [=] { - if (!IsValidIdentifierName(name)) { - throw std::invalid_argument{fmt::format("Invalid function name: {}", name)}; - } - if (module->functions.find(name) != module->functions.end()) { - throw std::logic_error{fmt::format("Function with similar name already exists '{}'", name)}; - }; - auto *memory = module->functions.get_allocator().GetMemoryResource(); - - return &module->functions.emplace(name, mgp_func(name, cb, memory)).first->second; - }, - result); -} diff --git a/src/query/v2/procedure/mg_procedure_impl.hpp b/src/query/v2/procedure/mg_procedure_impl.hpp deleted file mode 100644 index d1e9bb152..000000000 --- a/src/query/v2/procedure/mg_procedure_impl.hpp +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/// @file -/// Contains private (implementation) declarations and definitions for -/// mg_procedure.h -#pragma once - -#include "mg_procedure.h" - -#include -#include - -#include "integrations/kafka/consumer.hpp" -#include "integrations/pulsar/consumer.hpp" -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/context.hpp" -#include "query/v2/db_accessor.hpp" -#include "query/v2/frontend/ast/ast.hpp" -#include "query/v2/procedure/cypher_type_ptr.hpp" -#include "storage/v3/view.hpp" -#include "utils/memory.hpp" -#include "utils/pmr/map.hpp" -#include "utils/pmr/string.hpp" -#include "utils/pmr/vector.hpp" -#include "utils/temporal.hpp" -/// Wraps memory resource used in custom procedures. -/// -/// This should have been `using mgp_memory = memgraph::utils::MemoryResource`, but that's -/// not valid C++ because we have a forward declare `struct mgp_memory` in -/// mg_procedure.h -/// TODO: Make this extendable in C API, so that custom procedure writer can add -/// their own memory management wrappers. -struct mgp_memory { - memgraph::utils::MemoryResource *impl; -}; - -/// Immutable container of various values that appear in openCypher. -struct mgp_value { - /// Allocator type so that STL containers are aware that we need one. - using allocator_type = memgraph::utils::Allocator; - - // Construct MGP_VALUE_TYPE_NULL. - explicit mgp_value(memgraph::utils::MemoryResource *) noexcept; - - mgp_value(bool, memgraph::utils::MemoryResource *) noexcept; - mgp_value(int64_t, memgraph::utils::MemoryResource *) noexcept; - mgp_value(double, memgraph::utils::MemoryResource *) noexcept; - /// @throw std::bad_alloc - mgp_value(const char *, memgraph::utils::MemoryResource *); - /// Take ownership of the mgp_list, MemoryResource must match. - mgp_value(mgp_list *, memgraph::utils::MemoryResource *) noexcept; - /// Take ownership of the mgp_map, MemoryResource must match. - mgp_value(mgp_map *, memgraph::utils::MemoryResource *) noexcept; - /// Take ownership of the mgp_vertex, MemoryResource must match. - mgp_value(mgp_vertex *, memgraph::utils::MemoryResource *) noexcept; - /// Take ownership of the mgp_edge, MemoryResource must match. - mgp_value(mgp_edge *, memgraph::utils::MemoryResource *) noexcept; - /// Take ownership of the mgp_path, MemoryResource must match. - mgp_value(mgp_path *, memgraph::utils::MemoryResource *) noexcept; - - mgp_value(mgp_date *, memgraph::utils::MemoryResource *) noexcept; - mgp_value(mgp_local_time *, memgraph::utils::MemoryResource *) noexcept; - mgp_value(mgp_local_date_time *, memgraph::utils::MemoryResource *) noexcept; - mgp_value(mgp_duration *, memgraph::utils::MemoryResource *) noexcept; - - /// Construct by copying memgraph::query::v2::TypedValue using memgraph::utils::MemoryResource. - /// mgp_graph is needed to construct mgp_vertex and mgp_edge. - /// @throw std::bad_alloc - mgp_value(const memgraph::query::v2::TypedValue &, mgp_graph *, memgraph::utils::MemoryResource *); - - /// Construct by copying memgraph::storage::v3::PropertyValue using memgraph::utils::MemoryResource. - /// @throw std::bad_alloc - mgp_value(const memgraph::storage::v3::PropertyValue &, memgraph::utils::MemoryResource *); - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_value(const mgp_value &) = delete; - - /// Copy construct using given memgraph::utils::MemoryResource. - /// @throw std::bad_alloc - mgp_value(const mgp_value &, memgraph::utils::MemoryResource *); - - /// Move construct using given memgraph::utils::MemoryResource. - /// @throw std::bad_alloc if MemoryResource is different, so we cannot move. - mgp_value(mgp_value &&, memgraph::utils::MemoryResource *); - - /// Move construct, memgraph::utils::MemoryResource is inherited. - mgp_value(mgp_value &&other) noexcept : mgp_value(other, other.memory) {} - - /// Copy-assignment is not allowed to preserve immutability. - mgp_value &operator=(const mgp_value &) = delete; - - /// Move-assignment is not allowed to preserve immutability. - mgp_value &operator=(mgp_value &&) = delete; - - ~mgp_value() noexcept; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - mgp_value_type type; - memgraph::utils::MemoryResource *memory; - - union { - bool bool_v; - int64_t int_v; - double double_v; - memgraph::utils::pmr::string string_v; - // We use pointers so that taking ownership via C API is easier. Besides, - // mgp_map cannot use incomplete mgp_value type, because that would be - // undefined behaviour. - mgp_list *list_v; - mgp_map *map_v; - mgp_vertex *vertex_v; - mgp_edge *edge_v; - mgp_path *path_v; - mgp_date *date_v; - mgp_local_time *local_time_v; - mgp_local_date_time *local_date_time_v; - mgp_duration *duration_v; - }; -}; - -inline memgraph::utils::DateParameters MapDateParameters(const mgp_date_parameters *parameters) { - return {.year = parameters->year, .month = parameters->month, .day = parameters->day}; -} - -struct mgp_date { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // Hopefully memgraph::utils::Date copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_copy_constructible_v); - - mgp_date(const memgraph::utils::Date &date, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), date(date) {} - - mgp_date(const std::string_view string, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), date(memgraph::utils::ParseDateParameters(string).first) {} - - mgp_date(const mgp_date_parameters *parameters, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), date(MapDateParameters(parameters)) {} - - mgp_date(const int64_t microseconds, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), date(microseconds) {} - - mgp_date(const mgp_date &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), date(other.date) {} - - mgp_date(mgp_date &&other, memgraph::utils::MemoryResource *memory) noexcept : memory(memory), date(other.date) {} - - mgp_date(mgp_date &&other) noexcept : memory(other.memory), date(other.date) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_date(const mgp_date &) = delete; - - mgp_date &operator=(const mgp_date &) = delete; - mgp_date &operator=(mgp_date &&) = delete; - - ~mgp_date() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::utils::Date date; -}; - -inline memgraph::utils::LocalTimeParameters MapLocalTimeParameters(const mgp_local_time_parameters *parameters) { - return {.hour = parameters->hour, - .minute = parameters->minute, - .second = parameters->second, - .millisecond = parameters->millisecond, - .microsecond = parameters->microsecond}; -} - -struct mgp_local_time { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // Hopefully memgraph::utils::LocalTime copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_copy_constructible_v); - - mgp_local_time(const std::string_view string, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(memgraph::utils::ParseLocalTimeParameters(string).first) {} - - mgp_local_time(const mgp_local_time_parameters *parameters, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(MapLocalTimeParameters(parameters)) {} - - mgp_local_time(const memgraph::utils::LocalTime &local_time, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(local_time) {} - - mgp_local_time(const int64_t microseconds, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(microseconds) {} - - mgp_local_time(const mgp_local_time &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(other.local_time) {} - - mgp_local_time(mgp_local_time &&other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_time(other.local_time) {} - - mgp_local_time(mgp_local_time &&other) noexcept : memory(other.memory), local_time(other.local_time) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_local_time(const mgp_local_time &) = delete; - - mgp_local_time &operator=(const mgp_local_time &) = delete; - mgp_local_time &operator=(mgp_local_time &&) = delete; - - ~mgp_local_time() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::utils::LocalTime local_time; -}; - -inline memgraph::utils::LocalDateTime CreateLocalDateTimeFromString(const std::string_view string) { - const auto &[date_parameters, local_time_parameters] = memgraph::utils::ParseLocalDateTimeParameters(string); - return memgraph::utils::LocalDateTime{date_parameters, local_time_parameters}; -} - -struct mgp_local_date_time { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // Hopefully memgraph::utils::LocalDateTime copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_copy_constructible_v); - - mgp_local_date_time(const memgraph::utils::LocalDateTime &local_date_time, - memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_date_time(local_date_time) {} - - mgp_local_date_time(const std::string_view string, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_date_time(CreateLocalDateTimeFromString(string)) {} - - mgp_local_date_time(const mgp_local_date_time_parameters *parameters, - memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), - local_date_time(MapDateParameters(parameters->date_parameters), - MapLocalTimeParameters(parameters->local_time_parameters)) {} - - mgp_local_date_time(const int64_t microseconds, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_date_time(microseconds) {} - - mgp_local_date_time(const mgp_local_date_time &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_date_time(other.local_date_time) {} - - mgp_local_date_time(mgp_local_date_time &&other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), local_date_time(other.local_date_time) {} - - mgp_local_date_time(mgp_local_date_time &&other) noexcept - : memory(other.memory), local_date_time(other.local_date_time) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_local_date_time(const mgp_local_date_time &) = delete; - - mgp_local_date_time &operator=(const mgp_local_date_time &) = delete; - mgp_local_date_time &operator=(mgp_local_date_time &&) = delete; - - ~mgp_local_date_time() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::utils::LocalDateTime local_date_time; -}; - -inline memgraph::utils::DurationParameters MapDurationParameters(const mgp_duration_parameters *parameters) { - return {.day = parameters->day, - .hour = parameters->hour, - .minute = parameters->minute, - .second = parameters->second, - .millisecond = parameters->millisecond, - .microsecond = parameters->microsecond}; -} - -struct mgp_duration { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // Hopefully memgraph::utils::Duration copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_copy_constructible_v); - - mgp_duration(const std::string_view string, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(memgraph::utils::ParseDurationParameters(string)) {} - - mgp_duration(const mgp_duration_parameters *parameters, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(MapDurationParameters(parameters)) {} - - mgp_duration(const memgraph::utils::Duration &duration, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(duration) {} - - mgp_duration(const int64_t microseconds, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(microseconds) {} - - mgp_duration(const mgp_duration &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(other.duration) {} - - mgp_duration(mgp_duration &&other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), duration(other.duration) {} - - mgp_duration(mgp_duration &&other) noexcept : memory(other.memory), duration(other.duration) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_duration(const mgp_duration &) = delete; - - mgp_duration &operator=(const mgp_duration &) = delete; - mgp_duration &operator=(mgp_duration &&) = delete; - - ~mgp_duration() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::utils::Duration duration; -}; - -struct mgp_list { - /// Allocator type so that STL containers are aware that we need one. - using allocator_type = memgraph::utils::Allocator; - - explicit mgp_list(memgraph::utils::MemoryResource *memory) : elems(memory) {} - - mgp_list(memgraph::utils::pmr::vector &&elems, memgraph::utils::MemoryResource *memory) - : elems(std::move(elems), memory) {} - - mgp_list(const mgp_list &other, memgraph::utils::MemoryResource *memory) : elems(other.elems, memory) {} - - mgp_list(mgp_list &&other, memgraph::utils::MemoryResource *memory) : elems(std::move(other.elems), memory) {} - - mgp_list(mgp_list &&other) noexcept : elems(std::move(other.elems)) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_list(const mgp_list &) = delete; - - mgp_list &operator=(const mgp_list &) = delete; - mgp_list &operator=(mgp_list &&) = delete; - - ~mgp_list() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { - return elems.get_allocator().GetMemoryResource(); - } - - // C++17 vector can work with incomplete type. - memgraph::utils::pmr::vector elems; -}; - -struct mgp_map { - /// Allocator type so that STL containers are aware that we need one. - using allocator_type = memgraph::utils::Allocator; - - explicit mgp_map(memgraph::utils::MemoryResource *memory) : items(memory) {} - - mgp_map(memgraph::utils::pmr::map &&items, - memgraph::utils::MemoryResource *memory) - : items(std::move(items), memory) {} - - mgp_map(const mgp_map &other, memgraph::utils::MemoryResource *memory) : items(other.items, memory) {} - - mgp_map(mgp_map &&other, memgraph::utils::MemoryResource *memory) : items(std::move(other.items), memory) {} - - mgp_map(mgp_map &&other) noexcept : items(std::move(other.items)) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_map(const mgp_map &) = delete; - - mgp_map &operator=(const mgp_map &) = delete; - mgp_map &operator=(mgp_map &&) = delete; - - ~mgp_map() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { - return items.get_allocator().GetMemoryResource(); - } - - // Unfortunately using incomplete type with map is undefined, so mgp_map - // needs to be defined after mgp_value. - memgraph::utils::pmr::map items; -}; - -struct mgp_map_item { - const char *key; - mgp_value *value; -}; - -struct mgp_map_items_iterator { - using allocator_type = memgraph::utils::Allocator; - - mgp_map_items_iterator(mgp_map *map, memgraph::utils::MemoryResource *memory) - : memory(memory), map(map), current_it(map->items.begin()) { - if (current_it != map->items.end()) { - current.key = current_it->first.c_str(); - current.value = ¤t_it->second; - } - } - - mgp_map_items_iterator(const mgp_map_items_iterator &) = delete; - mgp_map_items_iterator(mgp_map_items_iterator &&) = delete; - mgp_map_items_iterator &operator=(const mgp_map_items_iterator &) = delete; - mgp_map_items_iterator &operator=(mgp_map_items_iterator &&) = delete; - - ~mgp_map_items_iterator() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const { return memory; } - - memgraph::utils::MemoryResource *memory; - mgp_map *map; - decltype(map->items.begin()) current_it; - mgp_map_item current; -}; - -struct mgp_vertex { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // Hopefully VertexAccessor copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_copy_constructible_v); - - mgp_vertex(memgraph::query::v2::VertexAccessor v, mgp_graph *graph, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), impl(v), graph(graph) {} - - mgp_vertex(const mgp_vertex &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), impl(other.impl), graph(other.graph) {} - - mgp_vertex(mgp_vertex &&other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), impl(other.impl), graph(other.graph) {} - - mgp_vertex(mgp_vertex &&other) noexcept : memory(other.memory), impl(other.impl), graph(other.graph) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_vertex(const mgp_vertex &) = delete; - - mgp_vertex &operator=(const mgp_vertex &) = delete; - mgp_vertex &operator=(mgp_vertex &&) = delete; - - bool operator==(const mgp_vertex &other) const noexcept { return this->impl == other.impl; } - bool operator!=(const mgp_vertex &other) const noexcept { return !(*this == other); }; - - ~mgp_vertex() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::query::v2::VertexAccessor impl; - mgp_graph *graph; -}; - -struct mgp_edge { - /// Allocator type so that STL containers are aware that we need one. - /// We don't actually need this, but it simplifies the C API, because we store - /// the allocator which was used to allocate `this`. - using allocator_type = memgraph::utils::Allocator; - - // TODO(antaljanosbenjamin): Handle this static assert failure when we will support procedures again - // Hopefully EdgeAccessor copy constructor remains noexcept, so that we can - // have everything noexcept here. - // static_assert(std::is_nothrow_copy_constructible_v); - - static mgp_edge *Copy(const mgp_edge &edge, mgp_memory &memory); - - mgp_edge(const memgraph::query::v2::EdgeAccessor &impl, mgp_graph *graph, - memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), impl(impl), from(impl.From(), graph, memory), to(impl.To(), graph, memory) {} - - mgp_edge(const mgp_edge &other, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), impl(other.impl), from(other.from, memory), to(other.to, memory) {} - - mgp_edge(mgp_edge &&other, memgraph::utils::MemoryResource *memory) noexcept - : memory(other.memory), impl(other.impl), from(std::move(other.from), memory), to(std::move(other.to), memory) {} - - mgp_edge(mgp_edge &&other) noexcept - : memory(other.memory), impl(other.impl), from(std::move(other.from)), to(std::move(other.to)) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_edge(const mgp_edge &) = delete; - - mgp_edge &operator=(const mgp_edge &) = delete; - mgp_edge &operator=(mgp_edge &&) = delete; - ~mgp_edge() = default; - - bool operator==(const mgp_edge &other) const noexcept { return this->impl == other.impl; } - bool operator!=(const mgp_edge &other) const noexcept { return !(*this == other); }; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { return memory; } - - memgraph::utils::MemoryResource *memory; - memgraph::query::v2::EdgeAccessor impl; - mgp_vertex from; - mgp_vertex to; -}; - -struct mgp_path { - /// Allocator type so that STL containers are aware that we need one. - using allocator_type = memgraph::utils::Allocator; - - explicit mgp_path(memgraph::utils::MemoryResource *memory) : vertices(memory), edges(memory) {} - - mgp_path(const mgp_path &other, memgraph::utils::MemoryResource *memory) - : vertices(other.vertices, memory), edges(other.edges, memory) {} - - mgp_path(mgp_path &&other, memgraph::utils::MemoryResource *memory) - : vertices(std::move(other.vertices), memory), edges(std::move(other.edges), memory) {} - - mgp_path(mgp_path &&other) noexcept : vertices(std::move(other.vertices)), edges(std::move(other.edges)) {} - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - mgp_path(const mgp_path &) = delete; - - mgp_path &operator=(const mgp_path &) = delete; - mgp_path &operator=(mgp_path &&) = delete; - - ~mgp_path() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { - return vertices.get_allocator().GetMemoryResource(); - } - - memgraph::utils::pmr::vector vertices; - memgraph::utils::pmr::vector edges; -}; - -struct mgp_result_record { - /// Result record signature as defined for mgp_proc. - const memgraph::utils::pmr::map> *signature; - memgraph::utils::pmr::map values; -}; - -struct mgp_result { - explicit mgp_result( - const memgraph::utils::pmr::map> *signature, - memgraph::utils::MemoryResource *mem) - : signature(signature), rows(mem) {} - - /// Result record signature as defined for mgp_proc. - const memgraph::utils::pmr::map> *signature; - memgraph::utils::pmr::vector rows; - std::optional error_msg; -}; - -struct mgp_func_result { - mgp_func_result() {} - /// Return Magic function result. If user forgets it, the error is raised - std::optional value; - /// Return Magic function result with potential error - std::optional error_msg; -}; - -struct mgp_graph { - memgraph::query::v2::DbAccessor *impl; - memgraph::storage::v3::View view; - // TODO: Merge `mgp_graph` and `mgp_memory` into a single `mgp_context`. The - // `ctx` field is out of place here. - memgraph::query::v2::ExecutionContext *ctx; - - static mgp_graph WritableGraph(memgraph::query::v2::DbAccessor &acc, memgraph::storage::v3::View view, - memgraph::query::v2::ExecutionContext &ctx) { - return mgp_graph{&acc, view, &ctx}; - } - - static mgp_graph NonWritableGraph(memgraph::query::v2::DbAccessor &acc, memgraph::storage::v3::View view) { - return mgp_graph{&acc, view, nullptr}; - } -}; - -// Prevents user to use ExecutionContext in writable callables -struct mgp_func_context { - memgraph::query::v2::DbAccessor *impl; - memgraph::storage::v3::View view; -}; -struct mgp_properties_iterator { - using allocator_type = memgraph::utils::Allocator; - - // Define members at the start because we use decltype a lot here, so members - // need to be visible in method definitions. - - memgraph::utils::MemoryResource *memory; - mgp_graph *graph; - std::remove_reference_t().Properties(graph->view))> pvs; - decltype(pvs.begin()) current_it; - std::optional> current; - mgp_property property{nullptr, nullptr}; - - // Construct with no properties. - explicit mgp_properties_iterator(mgp_graph *graph, memgraph::utils::MemoryResource *memory) - : memory(memory), graph(graph), current_it(pvs.begin()) {} - - // May throw who the #$@! knows what because PropertyValueStore doesn't - // document what it throws, and it may surely throw some piece of !@#$ - // exception because it's built on top of STL and other libraries. - mgp_properties_iterator(mgp_graph *graph, decltype(pvs) pvs, memgraph::utils::MemoryResource *memory) - : memory(memory), graph(graph), pvs(std::move(pvs)), current_it(this->pvs.begin()) { - if (current_it != this->pvs.end()) { - current.emplace(memgraph::utils::pmr::string(graph->impl->PropertyToName(current_it->first), memory), - mgp_value(current_it->second, memory)); - property.name = current->first.c_str(); - property.value = ¤t->second; - } - } - - mgp_properties_iterator(const mgp_properties_iterator &) = delete; - mgp_properties_iterator(mgp_properties_iterator &&) = delete; - - mgp_properties_iterator &operator=(const mgp_properties_iterator &) = delete; - mgp_properties_iterator &operator=(mgp_properties_iterator &&) = delete; - - ~mgp_properties_iterator() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const { return memory; } -}; - -struct mgp_edges_iterator { - using allocator_type = memgraph::utils::Allocator; - - // Hopefully mgp_vertex copy constructor remains noexcept, so that we can - // have everything noexcept here. - static_assert(std::is_nothrow_constructible_v); - - mgp_edges_iterator(const mgp_vertex &v, memgraph::utils::MemoryResource *memory) noexcept - : memory(memory), source_vertex(v, memory) {} - - mgp_edges_iterator(mgp_edges_iterator &&other) noexcept - : memory(other.memory), - source_vertex(std::move(other.source_vertex)), - in(std::move(other.in)), - in_it(std::move(other.in_it)), - out(std::move(other.out)), - out_it(std::move(other.out_it)), - current_e(std::move(other.current_e)) {} - - mgp_edges_iterator(const mgp_edges_iterator &) = delete; - mgp_edges_iterator &operator=(const mgp_edges_iterator &) = delete; - mgp_edges_iterator &operator=(mgp_edges_iterator &&) = delete; - - ~mgp_edges_iterator() = default; - - memgraph::utils::MemoryResource *GetMemoryResource() const { return memory; } - - memgraph::utils::MemoryResource *memory; - mgp_vertex source_vertex; - std::optionalview))>> in; - std::optionalbegin())> in_it; - std::optionalview))>> out; - std::optionalbegin())> out_it; - std::optional current_e; -}; - -struct mgp_vertices_iterator { - using allocator_type = memgraph::utils::Allocator; - - /// @throw anything VerticesIterable may throw - mgp_vertices_iterator(mgp_graph *graph, memgraph::utils::MemoryResource *memory) - : memory(memory), graph(graph), vertices(graph->impl->Vertices(graph->view)), current_it(vertices.begin()) { - if (current_it != vertices.end()) { - current_v.emplace(*current_it, graph, memory); - } - } - - memgraph::utils::MemoryResource *GetMemoryResource() const { return memory; } - - memgraph::utils::MemoryResource *memory; - mgp_graph *graph; - decltype(graph->impl->Vertices(graph->view)) vertices; - decltype(vertices.begin()) current_it; - std::optional current_v; -}; - -struct mgp_type { - memgraph::query::v2::procedure::CypherTypePtr impl; -}; - -struct ProcedureInfo { - bool is_write = false; - std::optional required_privilege = std::nullopt; -}; -struct mgp_proc { - using allocator_type = memgraph::utils::Allocator; - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_proc(const char *name, mgp_proc_cb cb, memgraph::utils::MemoryResource *memory, const ProcedureInfo &info = {}) - : name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_proc(const char *name, std::function cb, - memgraph::utils::MemoryResource *memory, const ProcedureInfo &info = {}) - : name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_proc(const std::string_view name, std::function cb, - memgraph::utils::MemoryResource *memory, const ProcedureInfo &info = {}) - : name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_proc(const mgp_proc &other, memgraph::utils::MemoryResource *memory) - : name(other.name, memory), - cb(other.cb), - args(other.args, memory), - opt_args(other.opt_args, memory), - results(other.results, memory), - info(other.info) {} - - mgp_proc(mgp_proc &&other, memgraph::utils::MemoryResource *memory) - : name(std::move(other.name), memory), - cb(std::move(other.cb)), - args(std::move(other.args), memory), - opt_args(std::move(other.opt_args), memory), - results(std::move(other.results), memory), - info(other.info) {} - - mgp_proc(const mgp_proc &other) = default; - mgp_proc(mgp_proc &&other) = default; - - mgp_proc &operator=(const mgp_proc &) = delete; - mgp_proc &operator=(mgp_proc &&) = delete; - - ~mgp_proc() = default; - - /// Name of the procedure. - memgraph::utils::pmr::string name; - /// Entry-point for the procedure. - std::function cb; - /// Required, positional arguments as a (name, type) pair. - memgraph::utils::pmr::vector< - std::pair> - args; - /// Optional positional arguments as a (name, type, default_value) tuple. - memgraph::utils::pmr::vector< - std::tuple> - opt_args; - /// Fields this procedure returns, as a (name -> (type, is_deprecated)) map. - memgraph::utils::pmr::map> - results; - ProcedureInfo info; -}; - -struct mgp_trans { - using allocator_type = memgraph::utils::Allocator; - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_trans(const char *name, mgp_trans_cb cb, memgraph::utils::MemoryResource *memory) - : name(name, memory), cb(cb), results(memory) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_trans(const char *name, std::function cb, - memgraph::utils::MemoryResource *memory) - : name(name, memory), cb(cb), results(memory) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_trans(const mgp_trans &other, memgraph::utils::MemoryResource *memory) - : name(other.name, memory), cb(other.cb), results(other.results) {} - - mgp_trans(mgp_trans &&other, memgraph::utils::MemoryResource *memory) - : name(std::move(other.name), memory), cb(std::move(other.cb)), results(std::move(other.results)) {} - - mgp_trans(const mgp_trans &other) = default; - mgp_trans(mgp_trans &&other) = default; - - mgp_trans &operator=(const mgp_trans &) = delete; - mgp_trans &operator=(mgp_trans &&) = delete; - - ~mgp_trans() = default; - - /// Name of the transformation. - memgraph::utils::pmr::string name; - /// Entry-point for the transformation. - std::function cb; - /// Fields this transformation returns. - memgraph::utils::pmr::map> - results; -}; - -struct mgp_func { - using allocator_type = memgraph::utils::Allocator; - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_func(const char *name, mgp_func_cb cb, memgraph::utils::MemoryResource *memory) - : name(name, memory), cb(cb), args(memory), opt_args(memory) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_func(const char *name, std::function cb, - memgraph::utils::MemoryResource *memory) - : name(name, memory), cb(cb), args(memory), opt_args(memory) {} - - /// @throw std::bad_alloc - /// @throw std::length_error - mgp_func(const mgp_func &other, memgraph::utils::MemoryResource *memory) - : name(other.name, memory), cb(other.cb), args(other.args, memory), opt_args(other.opt_args, memory) {} - - mgp_func(mgp_func &&other, memgraph::utils::MemoryResource *memory) - : name(std::move(other.name), memory), - cb(std::move(other.cb)), - args(std::move(other.args), memory), - opt_args(std::move(other.opt_args), memory) {} - - mgp_func(const mgp_func &other) = default; - mgp_func(mgp_func &&other) = default; - - mgp_func &operator=(const mgp_func &) = delete; - mgp_func &operator=(mgp_func &&) = delete; - - ~mgp_func() = default; - - /// Name of the function. - memgraph::utils::pmr::string name; - /// Entry-point for the function. - std::function cb; - /// Required, positional arguments as a (name, type) pair. - memgraph::utils::pmr::vector< - std::pair> - args; - /// Optional positional arguments as a (name, type, default_value) tuple. - memgraph::utils::pmr::vector< - std::tuple> - opt_args; -}; - -mgp_error MgpTransAddFixedResult(mgp_trans *trans) noexcept; - -struct mgp_module { - using allocator_type = memgraph::utils::Allocator; - - explicit mgp_module(memgraph::utils::MemoryResource *memory) - : procedures(memory), transformations(memory), functions(memory) {} - - mgp_module(const mgp_module &other, memgraph::utils::MemoryResource *memory) - : procedures(other.procedures, memory), - transformations(other.transformations, memory), - functions(other.functions, memory) {} - - mgp_module(mgp_module &&other, memgraph::utils::MemoryResource *memory) - : procedures(std::move(other.procedures), memory), - transformations(std::move(other.transformations), memory), - functions(std::move(other.functions), memory) {} - - mgp_module(const mgp_module &) = default; - mgp_module(mgp_module &&) = default; - - mgp_module &operator=(const mgp_module &) = delete; - mgp_module &operator=(mgp_module &&) = delete; - - ~mgp_module() = default; - - memgraph::utils::pmr::map procedures; - memgraph::utils::pmr::map transformations; - memgraph::utils::pmr::map functions; -}; - -namespace memgraph::query::v2::procedure { - -/// @throw std::bad_alloc -/// @throw std::length_error -/// @throw anything std::ostream::operator<< may throw. -void PrintProcSignature(const mgp_proc &, std::ostream *); - -/// @throw std::bad_alloc -/// @throw std::length_error -/// @throw anything std::ostream::operator<< may throw. -void PrintFuncSignature(const mgp_func &, std::ostream &); - -bool IsValidIdentifierName(const char *name); - -} // namespace memgraph::query::v2::procedure - -struct mgp_message { - explicit mgp_message(const memgraph::integrations::kafka::Message &message) : msg{&message} {} - explicit mgp_message(const memgraph::integrations::pulsar::Message &message) : msg{message} {} - - using KafkaMessage = const memgraph::integrations::kafka::Message *; - using PulsarMessage = memgraph::integrations::pulsar::Message; - std::variant msg; -}; - -struct mgp_messages { - using allocator_type = memgraph::utils::Allocator; - using storage_type = memgraph::utils::pmr::vector; - explicit mgp_messages(storage_type &&storage) : messages(std::move(storage)) {} - - mgp_messages(const mgp_messages &) = delete; - mgp_messages &operator=(const mgp_messages &) = delete; - - mgp_messages(mgp_messages &&) = delete; - mgp_messages &operator=(mgp_messages &&) = delete; - - ~mgp_messages() = default; - - storage_type messages; -}; - -memgraph::query::v2::TypedValue ToTypedValue(const mgp_value &val, memgraph::utils::MemoryResource *memory); diff --git a/src/query/v2/procedure/module.cpp b/src/query/v2/procedure/module.cpp deleted file mode 100644 index 038df8fa8..000000000 --- a/src/query/v2/procedure/module.cpp +++ /dev/null @@ -1,1258 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/procedure/module.hpp" - -#include -#include - -extern "C" { -#include -} - -#include -#include - -#include "py/py.hpp" -#include "query/v2/procedure/mg_procedure_helpers.hpp" -#include "query/v2/procedure/py_module.hpp" -#include "utils/file.hpp" -#include "utils/logging.hpp" -#include "utils/memory.hpp" -#include "utils/message.hpp" -#include "utils/pmr/vector.hpp" -#include "utils/string.hpp" - -namespace memgraph::query::v2::procedure { - -ModuleRegistry gModuleRegistry; - -Module::~Module() {} - -class BuiltinModule final : public Module { - public: - BuiltinModule(); - ~BuiltinModule() override; - BuiltinModule(const BuiltinModule &) = delete; - BuiltinModule(BuiltinModule &&) = delete; - BuiltinModule &operator=(const BuiltinModule &) = delete; - BuiltinModule &operator=(BuiltinModule &&) = delete; - - bool Close() override; - - const std::map> *Procedures() const override; - - const std::map> *Transformations() const override; - - const std::map> *Functions() const override; - - void AddProcedure(std::string_view name, mgp_proc proc); - - void AddTransformation(std::string_view name, mgp_trans trans); - - std::optional Path() const override { return std::nullopt; } - - private: - /// Registered procedures - std::map> procedures_; - std::map> transformations_; - std::map> functions_; -}; - -BuiltinModule::BuiltinModule() {} - -BuiltinModule::~BuiltinModule() {} - -bool BuiltinModule::Close() { return true; } - -const std::map> *BuiltinModule::Procedures() const { return &procedures_; } - -const std::map> *BuiltinModule::Transformations() const { - return &transformations_; -} -const std::map> *BuiltinModule::Functions() const { return &functions_; } - -void BuiltinModule::AddProcedure(std::string_view name, mgp_proc proc) { procedures_.emplace(name, std::move(proc)); } - -void BuiltinModule::AddTransformation(std::string_view name, mgp_trans trans) { - transformations_.emplace(name, std::move(trans)); -} - -namespace { - -auto WithUpgradedLock(auto *lock, const auto &function) { - lock->unlock_shared(); - utils::OnScopeExit shared_lock{[&] { lock->lock_shared(); }}; - function(); -}; - -void RegisterMgLoad(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) { - // Loading relies on the fact that regular procedure invocation through - // CallProcedureCursor::Pull takes ModuleRegistry::lock_ with READ access. To - // load modules we have to upgrade our READ access to WRITE access, - // therefore we release the READ lock and invoke the load function which - // takes the WRITE lock. Obviously, some other thread may take a READ or WRITE - // lock during our transition when we hold no such lock. In this case it is - // fine, because our builtin module cannot be unloaded and we are ok with - // using the new state of module_registry when we manage to acquire the lock - // we desire. Note, deadlock between threads should not be possible, because a - // single thread may only take either a READ or a WRITE lock, it's not - // possible for a thread to hold both. If a thread tries to do that, it will - // deadlock immediately (no other thread needs to do anything). - auto load_all_cb = [module_registry, lock](mgp_list * /*args*/, mgp_graph * /*graph*/, mgp_result * /*result*/, - mgp_memory * /*memory*/) { - WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); }); - }; - mgp_proc load_all("load_all", load_all_cb, utils::NewDeleteResource()); - module->AddProcedure("load_all", std::move(load_all)); - auto load_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result, - mgp_memory * /*memory*/) { - MG_ASSERT(Call(mgp_list_size, args) == 1U, "Should have been type checked already"); - auto *arg = Call(mgp_list_at, args, 0); - MG_ASSERT(CallBool(mgp_value_is_string, arg), "Should have been type checked already"); - bool succ = false; - WithUpgradedLock(lock, [&]() { - const char *arg_as_string{nullptr}; - if (const auto err = mgp_value_get_string(arg, &arg_as_string); err != mgp_error::MGP_ERROR_NO_ERROR) { - succ = false; - } else { - succ = module_registry->LoadOrReloadModuleFromName(arg_as_string); - } - }); - if (!succ) { - MG_ASSERT(mgp_result_set_error_msg(result, "Failed to (re)load the module.") == mgp_error::MGP_ERROR_NO_ERROR); - } - }; - mgp_proc load("load", load_cb, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_arg(&load, "module_name", Call(mgp_type_string)) == mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("load", std::move(load)); -} - -namespace { -[[nodiscard]] bool IsFileEditable(const std::optional &path) { - return path && access(path->c_str(), W_OK) == 0; -} - -std::string GetPathString(const std::optional &path) { - if (!path) { - return "builtin"; - } - - return std::filesystem::canonical(*path).generic_string(); -} -} // namespace - -void RegisterMgProcedures( - // We expect modules to be sorted by name. - const std::map, std::less<>> *all_modules, BuiltinModule *module) { - auto procedures_cb = [all_modules](mgp_list * /*args*/, mgp_graph * /*graph*/, mgp_result *result, - mgp_memory *memory) { - // Iterating over all_modules assumes that the standard mechanism of custom - // procedure invocations takes the ModuleRegistry::lock_ with READ access. - // For details on how the invocation is done, take a look at the - // CallProcedureCursor::Pull implementation. - for (const auto &[module_name, module] : *all_modules) { - // Return the results in sorted order by module and by procedure. - static_assert( - std::is_same_vProcedures()), const std::map> *>, - "Expected module procedures to be sorted by name"); - - const auto path = module->Path(); - const auto path_string = GetPathString(path); - const auto is_editable = IsFileEditable(path); - - for (const auto &[proc_name, proc] : *module->Procedures()) { - mgp_result_record *record{nullptr}; - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result); - if (!path_value) { - return; - } - - MgpUniquePtr is_editable_value{nullptr, mgp_value_destroy}; - if (!TryOrSetError([&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); }, - result)) { - return; - } - - utils::pmr::string full_name(module_name, memory->impl); - full_name.append(1, '.'); - full_name.append(proc_name); - const auto name_value = GetStringValueOrSetError(full_name.c_str(), memory, result); - if (!name_value) { - return; - } - - std::stringstream ss; - ss << module_name << "."; - PrintProcSignature(proc, &ss); - const auto signature = ss.str(); - const auto signature_value = GetStringValueOrSetError(signature.c_str(), memory, result); - if (!signature_value) { - return; - } - - MgpUniquePtr is_write_value{nullptr, mgp_value_destroy}; - if (!TryOrSetError( - [&, &proc = proc] { - return CreateMgpObject(is_write_value, mgp_value_make_bool, proc.info.is_write ? 1 : 0, memory); - }, - result)) { - return; - } - - if (!InsertResultOrSetError(result, record, "name", name_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "signature", signature_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "is_write", is_write_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "path", path_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) { - return; - } - } - } - }; - mgp_proc procedures("procedures", procedures_cb, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_result(&procedures, "name", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "signature", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "is_write", Call(mgp_type_bool)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "is_editable", Call(mgp_type_bool)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("procedures", std::move(procedures)); -} - -void RegisterMgTransformations(const std::map, std::less<>> *all_modules, - BuiltinModule *module) { - auto transformations_cb = [all_modules](mgp_list * /*unused*/, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory *memory) { - for (const auto &[module_name, module] : *all_modules) { - // Return the results in sorted order by module and by transformation. - static_assert( - std::is_same_vTransformations()), const std::map> *>, - "Expected module transformations to be sorted by name"); - - const auto path = module->Path(); - const auto path_string = GetPathString(path); - const auto is_editable = IsFileEditable(path); - - for (const auto &[trans_name, proc] : *module->Transformations()) { - mgp_result_record *record{nullptr}; - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result); - if (!path_value) { - return; - } - - MgpUniquePtr is_editable_value{nullptr, mgp_value_destroy}; - if (!TryOrSetError([&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); }, - result)) { - return; - } - - utils::pmr::string full_name(module_name, memory->impl); - full_name.append(1, '.'); - full_name.append(trans_name); - - const auto name_value = GetStringValueOrSetError(full_name.c_str(), memory, result); - if (!name_value) { - return; - } - - if (!InsertResultOrSetError(result, record, "name", name_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "path", path_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) { - return; - } - } - } - }; - mgp_proc procedures("transformations", transformations_cb, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_result(&procedures, "name", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&procedures, "is_editable", Call(mgp_type_bool)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("transformations", std::move(procedures)); -} - -void RegisterMgFunctions( - // We expect modules to be sorted by name. - const std::map, std::less<>> *all_modules, BuiltinModule *module) { - auto functions_cb = [all_modules](mgp_list * /*args*/, mgp_graph * /*graph*/, mgp_result *result, - mgp_memory *memory) { - // Iterating over all_modules assumes that the standard mechanism of magic - // functions invocations takes the ModuleRegistry::lock_ with READ access. - for (const auto &[module_name, module] : *all_modules) { - // Return the results in sorted order by module and by function_name. - static_assert(std::is_same_vFunctions()), const std::map> *>, - "Expected module magic functions to be sorted by name"); - - const auto path = module->Path(); - const auto path_string = GetPathString(path); - const auto is_editable = IsFileEditable(path); - - for (const auto &[func_name, func] : *module->Functions()) { - mgp_result_record *record{nullptr}; - - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result); - if (!path_value) { - return; - } - - MgpUniquePtr is_editable_value{nullptr, mgp_value_destroy}; - if (!TryOrSetError([&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); }, - result)) { - return; - } - - utils::pmr::string full_name(module_name, memory->impl); - full_name.append(1, '.'); - full_name.append(func_name); - const auto name_value = GetStringValueOrSetError(full_name.c_str(), memory, result); - if (!name_value) { - return; - } - - std::stringstream ss; - ss << module_name << "."; - PrintFuncSignature(func, ss); - const auto signature = ss.str(); - const auto signature_value = GetStringValueOrSetError(signature.c_str(), memory, result); - if (!signature_value) { - return; - } - - if (!InsertResultOrSetError(result, record, "name", name_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "signature", signature_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "path", path_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) { - return; - } - } - } - }; - mgp_proc functions("functions", functions_cb, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_result(&functions, "name", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&functions, "signature", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&functions, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&functions, "is_editable", Call(mgp_type_bool)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("functions", std::move(functions)); -} -namespace { -bool IsAllowedExtension(const auto &extension) { - static constexpr std::array allowed_extensions{".py"}; - return std::any_of(allowed_extensions.begin(), allowed_extensions.end(), - [&](const auto allowed_extension) { return allowed_extension == extension; }); -} - -bool IsSubPath(const auto &base, const auto &destination) { - const auto relative = std::filesystem::relative(destination, base); - return !relative.empty() && *relative.begin() != ".."; -} - -std::optional ReadFile(const auto &path) { - std::ifstream file(path); - if (!file.is_open()) { - return std::nullopt; - } - - const auto size = std::filesystem::file_size(path); - std::string content(size, '\0'); - file.read(content.data(), static_cast(size)); - return std::move(content); -} - -// Return the module directory that contains the `path` -utils::BasicResult ParentModuleDirectory(const ModuleRegistry &module_registry, - const std::filesystem::path &path) { - const auto &module_directories = module_registry.GetModulesDirectory(); - - auto longest_parent_directory = module_directories.end(); - auto max_length = std::numeric_limits::min(); - for (auto it = module_directories.begin(); it != module_directories.end(); ++it) { - if (IsSubPath(*it, path)) { - const auto length = std::filesystem::canonical(*it).string().size(); - if (length > max_length) { - longest_parent_directory = it; - max_length = length; - } - } - } - - if (longest_parent_directory == module_directories.end()) { - return "The specified file isn't contained in any of the module directories."; - } - - return *longest_parent_directory; -} -} // namespace - -void RegisterMgGetModuleFiles(ModuleRegistry *module_registry, BuiltinModule *module) { - auto get_module_files_cb = [module_registry](mgp_list * /*args*/, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory *memory) { - for (const auto &module_directory : module_registry->GetModulesDirectory()) { - for (const auto &dir_entry : std::filesystem::recursive_directory_iterator(module_directory)) { - if (dir_entry.is_regular_file() && IsAllowedExtension(dir_entry.path().extension())) { - mgp_result_record *record{nullptr}; - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto path_string = GetPathString(dir_entry); - const auto is_editable = IsFileEditable(dir_entry); - - const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result); - if (!path_value) { - return; - } - - MgpUniquePtr is_editable_value{nullptr, mgp_value_destroy}; - if (!TryOrSetError( - [&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); }, - result)) { - return; - } - - if (!InsertResultOrSetError(result, record, "path", path_value.get())) { - return; - } - - if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) { - return; - } - } - } - } - }; - - mgp_proc get_module_files("get_module_files", get_module_files_cb, utils::NewDeleteResource(), - {.required_privilege = AuthQuery::Privilege::MODULE_READ}); - MG_ASSERT(mgp_proc_add_result(&get_module_files, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&get_module_files, "is_editable", Call(mgp_type_bool)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("get_module_files", std::move(get_module_files)); -} - -void RegisterMgGetModuleFile(ModuleRegistry *module_registry, BuiltinModule *module) { - auto get_module_file_cb = [module_registry](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory *memory) { - MG_ASSERT(Call(mgp_list_size, args) == 1U, "Should have been type checked already"); - auto *arg = Call(mgp_list_at, args, 0); - MG_ASSERT(CallBool(mgp_value_is_string, arg), "Should have been type checked already"); - const char *path_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(arg, &path_str); }, result)) { - return; - } - - const std::filesystem::path path{path_str}; - - if (!path.is_absolute()) { - static_cast(mgp_result_set_error_msg(result, "The path should be an absolute path.")); - return; - } - - if (!IsAllowedExtension(path.extension())) { - static_cast(mgp_result_set_error_msg(result, "The specified file isn't in the supported format.")); - return; - } - - if (!std::filesystem::exists(path)) { - static_cast(mgp_result_set_error_msg(result, "The specified file doesn't exist.")); - return; - } - - if (auto maybe_error_msg = ParentModuleDirectory(*module_registry, path); maybe_error_msg.HasError()) { - static_cast(mgp_result_set_error_msg(result, maybe_error_msg.GetError())); - return; - } - - const auto maybe_content = ReadFile(path); - if (!maybe_content) { - static_cast(mgp_result_set_error_msg(result, "Couldn't read the content of the file.")); - return; - } - - mgp_result_record *record{nullptr}; - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto content_value = GetStringValueOrSetError(maybe_content->c_str(), memory, result); - if (!content_value) { - return; - } - - if (!InsertResultOrSetError(result, record, "content", content_value.get())) { - return; - } - }; - mgp_proc get_module_file("get_module_file", std::move(get_module_file_cb), utils::NewDeleteResource(), - {.required_privilege = AuthQuery::Privilege::MODULE_READ}); - MG_ASSERT(mgp_proc_add_arg(&get_module_file, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&get_module_file, "content", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("get_module_file", std::move(get_module_file)); -} - -namespace { -utils::BasicResult WriteToFile(const std::filesystem::path &file, const std::string_view content) { - std::ofstream output_file{file}; - if (!output_file.is_open()) { - return fmt::format("Failed to open the file at location {}", file); - } - output_file.write(content.data(), static_cast(content.size())); - output_file.flush(); - return {}; -} -} // namespace - -void RegisterMgCreateModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) { - auto create_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory *memory) { - MG_ASSERT(Call(mgp_list_size, args) == 2U, "Should have been type checked already"); - auto *filename_arg = Call(mgp_list_at, args, 0); - MG_ASSERT(CallBool(mgp_value_is_string, filename_arg), "Should have been type checked already"); - const char *filename_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(filename_arg, &filename_str); }, result)) { - return; - } - - const auto file_path = module_registry->InternalModuleDir() / filename_str; - - if (!IsSubPath(module_registry->InternalModuleDir(), file_path)) { - static_cast(mgp_result_set_error_msg( - result, - "Invalid relative path defined. The module file cannot be define outside the internal modules directory.")); - return; - } - - if (!IsAllowedExtension(file_path.extension())) { - static_cast(mgp_result_set_error_msg(result, "The specified file isn't in the supported format.")); - return; - } - - if (std::filesystem::exists(file_path)) { - static_cast(mgp_result_set_error_msg(result, "File with the same name already exists!")); - return; - } - - utils::EnsureDir(file_path.parent_path()); - - auto *content_arg = Call(mgp_list_at, args, 1); - MG_ASSERT(CallBool(mgp_value_is_string, content_arg), "Should have been type checked already"); - const char *content_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(content_arg, &content_str); }, result)) { - return; - } - - if (auto maybe_error = WriteToFile(file_path, {content_str, std::strlen(content_str)}); maybe_error.HasError()) { - static_cast(mgp_result_set_error_msg(result, maybe_error.GetError().c_str())); - return; - } - - mgp_result_record *record{nullptr}; - if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto path_value = GetStringValueOrSetError(std::filesystem::canonical(file_path).c_str(), memory, result); - if (!path_value) { - return; - } - - if (!InsertResultOrSetError(result, record, "path", path_value.get())) { - return; - } - - WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); }); - }; - mgp_proc create_module_file("create_module_file", std::move(create_module_file_cb), utils::NewDeleteResource(), - {.required_privilege = AuthQuery::Privilege::MODULE_WRITE}); - MG_ASSERT(mgp_proc_add_arg(&create_module_file, "filename", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_arg(&create_module_file, "content", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&create_module_file, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("create_module_file", std::move(create_module_file)); -} - -void RegisterMgUpdateModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) { - auto update_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory * /*memory*/) { - MG_ASSERT(Call(mgp_list_size, args) == 2U, "Should have been type checked already"); - auto *path_arg = Call(mgp_list_at, args, 0); - MG_ASSERT(CallBool(mgp_value_is_string, path_arg), "Should have been type checked already"); - const char *path_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(path_arg, &path_str); }, result)) { - return; - } - - const std::filesystem::path path{path_str}; - - if (!path.is_absolute()) { - static_cast(mgp_result_set_error_msg(result, "The path should be an absolute path.")); - return; - } - - if (!IsAllowedExtension(path.extension())) { - static_cast(mgp_result_set_error_msg(result, "The specified file isn't in the supported format.")); - return; - } - - if (!std::filesystem::exists(path)) { - static_cast(mgp_result_set_error_msg(result, "The specified file doesn't exist.")); - return; - } - - if (auto maybe_error_msg = ParentModuleDirectory(*module_registry, path); maybe_error_msg.HasError()) { - static_cast(mgp_result_set_error_msg(result, maybe_error_msg.GetError())); - return; - } - - auto *content_arg = Call(mgp_list_at, args, 1); - MG_ASSERT(CallBool(mgp_value_is_string, content_arg), "Should have been type checked already"); - const char *content_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(content_arg, &content_str); }, result)) { - return; - } - - if (auto maybe_error = WriteToFile(path, {content_str, std::strlen(content_str)}); maybe_error.HasError()) { - static_cast(mgp_result_set_error_msg(result, maybe_error.GetError().c_str())); - return; - } - - WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); }); - }; - mgp_proc update_module_file("update_module_file", std::move(update_module_file_cb), utils::NewDeleteResource(), - {.required_privilege = AuthQuery::Privilege::MODULE_WRITE}); - MG_ASSERT(mgp_proc_add_arg(&update_module_file, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_arg(&update_module_file, "content", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("update_module_file", std::move(update_module_file)); -} - -void RegisterMgDeleteModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) { - auto delete_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result, - mgp_memory * /*memory*/) { - MG_ASSERT(Call(mgp_list_size, args) == 1U, "Should have been type checked already"); - auto *path_arg = Call(mgp_list_at, args, 0); - MG_ASSERT(CallBool(mgp_value_is_string, path_arg), "Should have been type checked already"); - const char *path_str{nullptr}; - if (!TryOrSetError([&] { return mgp_value_get_string(path_arg, &path_str); }, result)) { - return; - } - - const std::filesystem::path path{path_str}; - - if (!path.is_absolute()) { - static_cast(mgp_result_set_error_msg(result, "The path should be an absolute path.")); - return; - } - - if (!IsAllowedExtension(path.extension())) { - static_cast(mgp_result_set_error_msg(result, "The specified file isn't in the supported format.")); - return; - } - - if (!std::filesystem::exists(path)) { - static_cast(mgp_result_set_error_msg(result, "The specified file doesn't exist.")); - return; - } - - const auto parent_module_directory = ParentModuleDirectory(*module_registry, path); - if (parent_module_directory.HasError()) { - static_cast(mgp_result_set_error_msg(result, parent_module_directory.GetError())); - return; - } - - std::error_code ec; - if (!std::filesystem::remove(path, ec)) { - static_cast( - mgp_result_set_error_msg(result, fmt::format("Failed to delete the module: {}", ec.message()).c_str())); - return; - } - - auto parent_path = path.parent_path(); - while (!std::filesystem::is_symlink(parent_path) && std::filesystem::is_empty(parent_path) && - !std::filesystem::equivalent(*parent_module_directory, parent_path)) { - std::filesystem::remove(parent_path); - parent_path = parent_path.parent_path(); - } - - WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); }); - }; - mgp_proc delete_module_file("delete_module_file", std::move(delete_module_file_cb), utils::NewDeleteResource(), - {.required_privilege = AuthQuery::Privilege::MODULE_WRITE}); - MG_ASSERT(mgp_proc_add_arg(&delete_module_file, "path", Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - module->AddProcedure("delete_module_file", std::move(delete_module_file)); -} - -// Run `fun` with `mgp_module *` and `mgp_memory *` arguments. If `fun` returned -// a `true` value, store the `mgp_module::procedures` and -// `mgp_module::transformations into `proc_map`. The return value of WithModuleRegistration -// is the same as that of `fun`. Note, the return value need only be convertible to `bool`, -// it does not have to be `bool` itself. -template -auto WithModuleRegistration(TProcMap *proc_map, TTransMap *trans_map, TFuncMap *func_map, const TFun &fun) { - // We probably don't need more than 256KB for module initialization. - static constexpr size_t stack_bytes = 256UL * 1024UL; - unsigned char stack_memory[stack_bytes]; - utils::MonotonicBufferResource monotonic_memory(stack_memory, stack_bytes); - mgp_memory memory{&monotonic_memory}; - mgp_module module_def{memory.impl}; - auto res = fun(&module_def, &memory); - if (res) { - // Copy procedures into resulting proc_map. - for (const auto &proc : module_def.procedures) proc_map->emplace(proc); - // Copy transformations into resulting trans_map. - for (const auto &trans : module_def.transformations) trans_map->emplace(trans); - // Copy functions into resulting func_map. - for (const auto &func : module_def.functions) func_map->emplace(func); - } - return res; -} - -} // namespace - -class SharedLibraryModule final : public Module { - public: - SharedLibraryModule(); - ~SharedLibraryModule() override; - SharedLibraryModule(const SharedLibraryModule &) = delete; - SharedLibraryModule(SharedLibraryModule &&) = delete; - SharedLibraryModule &operator=(const SharedLibraryModule &) = delete; - SharedLibraryModule &operator=(SharedLibraryModule &&) = delete; - - bool Load(const std::filesystem::path &file_path); - - bool Close() override; - - const std::map> *Procedures() const override; - - const std::map> *Transformations() const override; - - const std::map> *Functions() const override; - - std::optional Path() const override { return file_path_; } - - private: - /// Path as requested for loading the module from a library. - std::filesystem::path file_path_; - /// System handle to shared library. - void *handle_; - /// Required initialization function called on module load. - std::function init_fn_; - /// Optional shutdown function called on module unload. - std::function shutdown_fn_; - /// Registered procedures - std::map> procedures_; - /// Registered transformations - std::map> transformations_; - /// Registered functions - std::map> functions_; -}; - -SharedLibraryModule::SharedLibraryModule() : handle_(nullptr) {} - -SharedLibraryModule::~SharedLibraryModule() { - if (handle_) Close(); -} - -bool SharedLibraryModule::Load(const std::filesystem::path &file_path) { - MG_ASSERT(!handle_, "Attempting to load an already loaded module..."); - spdlog::info("Loading module {}...", file_path); - file_path_ = file_path; - dlerror(); // Clear any existing error. - // NOLINTNEXTLINE(hicpp-signed-bitwise) - handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND); - if (!handle_) { - spdlog::error( - utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules")); - return false; - } - // Get required mgp_init_module - init_fn_ = reinterpret_cast(dlsym(handle_, "mgp_init_module")); - char *dl_errored = dlerror(); - if (!init_fn_ || dl_errored) { - spdlog::error( - utils::MessageWithLink("Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules")); - dlclose(handle_); - handle_ = nullptr; - return false; - } - auto module_cb = [&](auto *module_def, auto *memory) { - // Run mgp_init_module which must succeed. - int init_res = init_fn_(module_def, memory); - auto with_error = [this](std::string_view error_msg) { - spdlog::error(error_msg); - dlclose(handle_); - handle_ = nullptr; - return false; - }; - - if (init_res != 0) { - const auto error = fmt::format("Unable to load module {}; mgp_init_module_returned {} ", file_path, init_res); - return with_error(error); - } - for (auto &trans : module_def->transformations) { - const bool success = mgp_error::MGP_ERROR_NO_ERROR == MgpTransAddFixedResult(&trans.second); - if (!success) { - const auto error = - fmt::format("Unable to add result to transformation in module {}; add result failed", file_path); - return with_error(error); - } - } - return true; - }; - if (!WithModuleRegistration(&procedures_, &transformations_, &functions_, module_cb)) { - return false; - } - // Get optional mgp_shutdown_module - shutdown_fn_ = reinterpret_cast(dlsym(handle_, "mgp_shutdown_module")); - dl_errored = dlerror(); - if (dl_errored) spdlog::warn("When loading module {}; {}", file_path, dl_errored); - spdlog::info("Loaded module {}", file_path); - return true; -} - -bool SharedLibraryModule::Close() { - MG_ASSERT(handle_, "Attempting to close a module that has not been loaded..."); - spdlog::info("Closing module {}...", file_path_); - // non-existent shutdown function is semantically the same as a shutdown - // function that does nothing. - int shutdown_res = 0; - if (shutdown_fn_) shutdown_res = shutdown_fn_(); - if (shutdown_res != 0) { - spdlog::warn("When closing module {}; mgp_shutdown_module returned {}", file_path_, shutdown_res); - } - if (dlclose(handle_) != 0) { - spdlog::error( - utils::MessageWithLink("Failed to close module {}; {}.", file_path_, dlerror(), "https://memgr.ph/modules")); - return false; - } - spdlog::info("Closed module {}", file_path_); - handle_ = nullptr; - procedures_.clear(); - return true; -} - -const std::map> *SharedLibraryModule::Procedures() const { - MG_ASSERT(handle_, - "Attempting to access procedures of a module that has not " - "been loaded..."); - return &procedures_; -} - -const std::map> *SharedLibraryModule::Transformations() const { - MG_ASSERT(handle_, - "Attempting to access procedures of a module that has not " - "been loaded..."); - return &transformations_; -} - -const std::map> *SharedLibraryModule::Functions() const { - MG_ASSERT(handle_, - "Attempting to access functions of a module that has not " - "been loaded..."); - return &functions_; -} - -class PythonModule final : public Module { - public: - PythonModule(); - ~PythonModule() override; - PythonModule(const PythonModule &) = delete; - PythonModule(PythonModule &&) = delete; - PythonModule &operator=(const PythonModule &) = delete; - PythonModule &operator=(PythonModule &&) = delete; - - bool Load(const std::filesystem::path &file_path); - - bool Close() override; - - const std::map> *Procedures() const override; - const std::map> *Transformations() const override; - const std::map> *Functions() const override; - std::optional Path() const override { return file_path_; } - - private: - std::filesystem::path file_path_; - py::Object py_module_; - std::map> procedures_; - std::map> transformations_; - std::map> functions_; -}; - -PythonModule::PythonModule() {} - -PythonModule::~PythonModule() { - if (py_module_) Close(); -} - -bool PythonModule::Load(const std::filesystem::path &file_path) { - MG_ASSERT(!py_module_, "Attempting to load an already loaded module..."); - spdlog::info("Loading module {}...", file_path); - file_path_ = file_path; - auto gil = py::EnsureGIL(); - auto maybe_exc = py::AppendToSysPath(file_path.parent_path().c_str()); - if (maybe_exc) { - spdlog::error( - utils::MessageWithLink("Unable to load module {}; {}.", file_path, *maybe_exc, "https://memgr.ph/modules")); - return false; - } - bool succ = true; - auto module_cb = [&](auto *module_def, auto * /*memory*/) { - auto result = ImportPyModule(file_path.stem().c_str(), module_def); - for (auto &trans : module_def->transformations) { - succ = MgpTransAddFixedResult(&trans.second) == mgp_error::MGP_ERROR_NO_ERROR; - if (!succ) { - return result; - } - }; - return result; - }; - py_module_ = WithModuleRegistration(&procedures_, &transformations_, &functions_, module_cb); - if (py_module_) { - spdlog::info("Loaded module {}", file_path); - - if (!succ) { - spdlog::error("Unable to add result to transformation"); - return false; - } - return true; - } - auto exc_info = py::FetchError().value(); - spdlog::error( - utils::MessageWithLink("Unable to load module {}; {}.", file_path, exc_info, "https://memgr.ph/modules")); - return false; -} - -bool PythonModule::Close() { - MG_ASSERT(py_module_, "Attempting to close a module that has not been loaded..."); - spdlog::info("Closing module {}...", file_path_); - // The procedures and transformations are closures which hold references to the Python callbacks. - // Releasing these references might result in deallocations so we need to take the GIL. - auto gil = py::EnsureGIL(); - procedures_.clear(); - transformations_.clear(); - functions_.clear(); - // Delete the module from the `sys.modules` directory so that the module will - // be properly imported if imported again. - py::Object sys(PyImport_ImportModule("sys")); - if (PyDict_DelItemString(sys.GetAttr("modules").Ptr(), file_path_.stem().c_str()) != 0) { - spdlog::warn("Failed to remove the module from sys.modules"); - py_module_ = py::Object(nullptr); - return false; - } - - // Remove the cached bytecode if it's present - std::filesystem::remove_all(file_path_.parent_path() / "__pycache__"); - py_module_ = py::Object(nullptr); - spdlog::info("Closed module {}", file_path_); - return true; -} - -const std::map> *PythonModule::Procedures() const { - MG_ASSERT(py_module_, - "Attempting to access procedures of a module that has " - "not been loaded..."); - return &procedures_; -} - -const std::map> *PythonModule::Transformations() const { - MG_ASSERT(py_module_, - "Attempting to access procedures of a module that has " - "not been loaded..."); - return &transformations_; -} - -const std::map> *PythonModule::Functions() const { - MG_ASSERT(py_module_, - "Attempting to access functions of a module that has " - "not been loaded..."); - return &functions_; -} -namespace { - -std::unique_ptr LoadModuleFromFile(const std::filesystem::path &path) { - const auto &ext = path.extension(); - if (ext != ".so" && ext != ".py") { - spdlog::warn(utils::MessageWithLink("Unknown query module file {}.", path, "https://memgr.ph/modules")); - return nullptr; - } - std::unique_ptr module; - if (path.extension() == ".so") { - auto lib_module = std::make_unique(); - if (!lib_module->Load(path)) return nullptr; - module = std::move(lib_module); - } else if (path.extension() == ".py") { - auto py_module = std::make_unique(); - if (!py_module->Load(path)) return nullptr; - module = std::move(py_module); - } - return module; -} - -} // namespace - -bool ModuleRegistry::RegisterModule(const std::string_view name, std::unique_ptr module) { - MG_ASSERT(!name.empty(), "Module name cannot be empty"); - MG_ASSERT(module, "Tried to register an invalid module"); - if (modules_.find(name) != modules_.end()) { - spdlog::error( - utils::MessageWithLink("Unable to overwrite an already loaded module {}.", name, "https://memgr.ph/modules")); - return false; - } - modules_.emplace(name, std::move(module)); - return true; -} - -void ModuleRegistry::DoUnloadAllModules() { - MG_ASSERT(modules_.find("mg") != modules_.end(), "Expected the builtin \"mg\" module to be present."); - // This is correct because the destructor will close each module. However, - // we don't want to unload the builtin "mg" module. - auto module = std::move(modules_["mg"]); - modules_.clear(); - modules_.emplace("mg", std::move(module)); -} - -ModuleRegistry::ModuleRegistry() { - auto module = std::make_unique(); - RegisterMgProcedures(&modules_, module.get()); - RegisterMgTransformations(&modules_, module.get()); - RegisterMgFunctions(&modules_, module.get()); - RegisterMgLoad(this, &lock_, module.get()); - RegisterMgGetModuleFiles(this, module.get()); - RegisterMgGetModuleFile(this, module.get()); - RegisterMgCreateModuleFile(this, &lock_, module.get()); - RegisterMgUpdateModuleFile(this, &lock_, module.get()); - RegisterMgDeleteModuleFile(this, &lock_, module.get()); - modules_.emplace("mg", std::move(module)); -} - -void ModuleRegistry::SetModulesDirectory(std::vector modules_dirs, - const std::filesystem::path &data_directory) { - internal_module_dir_ = data_directory / "internal_modules"; - utils::EnsureDirOrDie(internal_module_dir_); - modules_dirs_ = std::move(modules_dirs); - modules_dirs_.push_back(internal_module_dir_); -} - -const std::vector &ModuleRegistry::GetModulesDirectory() const { return modules_dirs_; } - -bool ModuleRegistry::LoadModuleIfFound(const std::filesystem::path &modules_dir, const std::string_view name) { - if (!utils::DirExists(modules_dir)) { - spdlog::error( - utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules")); - return false; - } - for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) { - const auto &path = entry.path(); - if (entry.is_regular_file() && path.stem() == name) { - auto module = LoadModuleFromFile(path); - if (!module) return false; - return RegisterModule(name, std::move(module)); - } - } - return false; -} - -bool ModuleRegistry::LoadOrReloadModuleFromName(const std::string_view name) { - if (modules_dirs_.empty()) return false; - if (name.empty()) return false; - std::unique_lock guard(lock_); - auto found_it = modules_.find(name); - if (found_it != modules_.end()) { - if (!found_it->second->Close()) { - spdlog::warn("Failed to close module {}", found_it->first); - } - modules_.erase(found_it); - } - - for (const auto &module_dir : modules_dirs_) { - if (LoadModuleIfFound(module_dir, name)) { - return true; - } - } - return false; -} - -void ModuleRegistry::LoadModulesFromDirectory(const std::filesystem::path &modules_dir) { - if (modules_dir.empty()) return; - if (!utils::DirExists(modules_dir)) { - spdlog::error( - utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules")); - return; - } - for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) { - const auto &path = entry.path(); - if (entry.is_regular_file()) { - std::string name = path.stem(); - if (name.empty()) continue; - auto module = LoadModuleFromFile(path); - if (!module) continue; - RegisterModule(name, std::move(module)); - } - } -} - -void ModuleRegistry::UnloadAndLoadModulesFromDirectories() { - std::unique_lock guard(lock_); - DoUnloadAllModules(); - for (const auto &module_dir : modules_dirs_) { - LoadModulesFromDirectory(module_dir); - } -} - -ModulePtr ModuleRegistry::GetModuleNamed(const std::string_view name) const { - std::shared_lock guard(lock_); - auto found_it = modules_.find(name); - if (found_it == modules_.end()) return nullptr; - return ModulePtr(found_it->second.get(), std::move(guard)); -} - -void ModuleRegistry::UnloadAllModules() { - std::unique_lock guard(lock_); - DoUnloadAllModules(); -} - -utils::MemoryResource &ModuleRegistry::GetSharedMemoryResource() noexcept { return *shared_; } - -bool ModuleRegistry::RegisterMgProcedure(const std::string_view name, mgp_proc proc) { - std::unique_lock guard(lock_); - if (auto module = modules_.find("mg"); module != modules_.end()) { - auto *builtin_module = dynamic_cast(module->second.get()); - builtin_module->AddProcedure(name, std::move(proc)); - return true; - } - return false; -} - -const std::filesystem::path &ModuleRegistry::InternalModuleDir() const noexcept { return internal_module_dir_; } - -namespace { - -/// This function returns a pair of either -// ModuleName | Prop -/// 1. -/// 2. -std::optional> FindModuleNameAndProp( - const ModuleRegistry &module_registry, std::string_view fully_qualified_name, utils::MemoryResource *memory) { - utils::pmr::vector name_parts(memory); - utils::Split(&name_parts, fully_qualified_name, "."); - if (name_parts.size() == 1U) return std::nullopt; - auto last_dot_pos = fully_qualified_name.find_last_of('.'); - MG_ASSERT(last_dot_pos != std::string_view::npos); - - const auto &module_name = fully_qualified_name.substr(0, last_dot_pos); - const auto &name = name_parts.back(); - return std::make_pair(module_name, name); -} - -template -concept ModuleProperties = utils::SameAsAnyOf; - -template -std::optional> MakePairIfPropFound(const ModuleRegistry &module_registry, - std::string_view fully_qualified_name, - utils::MemoryResource *memory) { - auto prop_fun = [](auto &module) { - if constexpr (std::is_same_v) { - return module->Procedures(); - } else if constexpr (std::is_same_v) { - return module->Transformations(); - } else if constexpr (std::is_same_v) { - return module->Functions(); - } - }; - auto result = FindModuleNameAndProp(module_registry, fully_qualified_name, memory); - if (!result) return std::nullopt; - auto [module_name, prop_name] = *result; - auto module = module_registry.GetModuleNamed(module_name); - if (!module) return std::nullopt; - auto *prop = prop_fun(module); - const auto &prop_it = prop->find(prop_name); - if (prop_it == prop->end()) return std::nullopt; - return std::make_pair(std::move(module), &prop_it->second); -} - -} // namespace - -std::optional> FindProcedure(const ModuleRegistry &module_registry, - std::string_view fully_qualified_procedure_name, - utils::MemoryResource *memory) { - return MakePairIfPropFound(module_registry, fully_qualified_procedure_name, memory); -} - -std::optional> FindTransformation( - const ModuleRegistry &module_registry, std::string_view fully_qualified_transformation_name, - utils::MemoryResource *memory) { - return MakePairIfPropFound(module_registry, fully_qualified_transformation_name, memory); -} - -std::optional> FindFunction(const ModuleRegistry &module_registry, - std::string_view fully_qualified_function_name, - utils::MemoryResource *memory) { - return MakePairIfPropFound(module_registry, fully_qualified_function_name, memory); -} - -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/module.hpp b/src/query/v2/procedure/module.hpp deleted file mode 100644 index 628f94c34..000000000 --- a/src/query/v2/procedure/module.hpp +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/// @file -/// API for loading and registering modules providing custom oC procedures -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "query/v2/procedure/cypher_types.hpp" -#include "query/v2/procedure/mg_procedure_impl.hpp" -#include "utils/memory.hpp" -#include "utils/rw_lock.hpp" - -class CypherMainVisitorTest; - -namespace memgraph::query::v2::procedure { - -class Module { - public: - Module() {} - virtual ~Module(); - Module(const Module &) = delete; - Module(Module &&) = delete; - Module &operator=(const Module &) = delete; - Module &operator=(Module &&) = delete; - - /// Invokes the (optional) shutdown function and closes the module. - virtual bool Close() = 0; - - /// Returns registered procedures of this module - virtual const std::map> *Procedures() const = 0; - /// Returns registered transformations of this module - virtual const std::map> *Transformations() const = 0; - // /// Returns registered functions of this module - virtual const std::map> *Functions() const = 0; - - virtual std::optional Path() const = 0; -}; - -/// Proxy for a registered Module, acquires a read lock from ModuleRegistry. -class ModulePtr final { - const Module *module_{nullptr}; - std::shared_lock lock_; - - public: - ModulePtr() = default; - ModulePtr(std::nullptr_t) {} - ModulePtr(const Module *module, std::shared_lock lock) : module_(module), lock_(std::move(lock)) {} - - explicit operator bool() const { return static_cast(module_); } - - const Module &operator*() const { return *module_; } - const Module *operator->() const { return module_; } -}; - -/// Thread-safe registration of modules from libraries, uses utils::RWLock. -class ModuleRegistry final { - friend CypherMainVisitorTest; - - std::map, std::less<>> modules_; - mutable utils::RWLock lock_{utils::RWLock::Priority::WRITE}; - std::unique_ptr shared_{std::make_unique()}; - - bool RegisterModule(std::string_view name, std::unique_ptr module); - - void DoUnloadAllModules(); - - /// Loads the module if it's in the modules_dir directory - /// @return Whether the module was loaded - bool LoadModuleIfFound(const std::filesystem::path &modules_dir, std::string_view name); - - void LoadModulesFromDirectory(const std::filesystem::path &modules_dir); - - public: - ModuleRegistry(); - - /// Set the modules directories that will be used when (re)loading modules. - void SetModulesDirectory(std::vector modules_dir, const std::filesystem::path &data_directory); - const std::vector &GetModulesDirectory() const; - - /// Atomically load or reload a module with a particular name from the given - /// directory. - /// - /// Takes a write lock. If the module exists it is reloaded. Otherwise, the - /// module is loaded from the file whose filename, without the extension, - /// matches the module's name. If multiple such files exist, only one is - /// chosen, in an unspecified manner. If loading of the chosen file fails, no - /// other files are tried. - /// - /// Return true if the module was loaded or reloaded successfully, false - /// otherwise. - bool LoadOrReloadModuleFromName(std::string_view name); - - /// Atomically unload all modules and then load all possible modules from the - /// set directories. - /// - /// Takes a write lock. - void UnloadAndLoadModulesFromDirectories(); - - /// Find a module with given name or return nullptr. - /// Takes a read lock. - ModulePtr GetModuleNamed(std::string_view name) const; - - /// Remove all loaded (non-builtin) modules. - /// Takes a write lock. - void UnloadAllModules(); - - /// Returns the shared memory allocator used by modules - utils::MemoryResource &GetSharedMemoryResource() noexcept; - - bool RegisterMgProcedure(std::string_view name, mgp_proc proc); - - const std::filesystem::path &InternalModuleDir() const noexcept; - - private: - class SharedLibraryHandle { - public: - SharedLibraryHandle(const std::string &shared_library, int mode) : handle_{dlopen(shared_library.c_str(), mode)} {} - SharedLibraryHandle(const SharedLibraryHandle &) = delete; - SharedLibraryHandle(SharedLibraryHandle &&) = delete; - SharedLibraryHandle operator=(const SharedLibraryHandle &) = delete; - SharedLibraryHandle operator=(SharedLibraryHandle &&) = delete; - - ~SharedLibraryHandle() { - if (handle_) { - dlclose(handle_); - } - } - - private: - void *handle_; - }; - -#if __has_feature(address_sanitizer) - // This is why we need RTLD_NODELETE and we must not use RTLD_DEEPBIND with - // ASAN: https://github.com/google/sanitizers/issues/89 - SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE}; -#else - // The reason behind opening share library during runtime is to avoid issues - // with loading symbols from stdlib. We have encounter issues with locale - // that cause std::cout not being printed and issues when python libraries - // would call stdlib (e.g. pytorch). - // The way that those issues were solved was - // by using RTLD_DEEPBIND. RTLD_DEEPBIND ensures that the lookup for the - // mentioned library will be first performed in the already existing binded - // libraries and then the global namespace. - // RTLD_DEEPBIND => https://linux.die.net/man/3/dlopen - SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND}; -#endif - std::vector modules_dirs_; - std::filesystem::path internal_module_dir_; -}; - -/// Single, global module registry. -extern ModuleRegistry gModuleRegistry; - -/// Return the ModulePtr and `mgp_proc *` of the found procedure after resolving -/// `fully_qualified_procedure_name`. `memory` is used for temporary allocations -/// inside this function. ModulePtr must be kept alive to make sure it won't be -/// unloaded. -std::optional> FindProcedure( - const ModuleRegistry &module_registry, std::string_view fully_qualified_procedure_name, - utils::MemoryResource *memory); - -/// Return the ModulePtr and `mgp_trans *` of the found transformation after resolving -/// `fully_qualified_transformation_name`. `memory` is used for temporary allocations -/// inside this function. ModulePtr must be kept alive to make sure it won't be -/// unloaded. -std::optional> FindTransformation( - const ModuleRegistry &module_registry, std::string_view fully_qualified_transformation_name, - utils::MemoryResource *memory); - -/// Return the ModulePtr and `mgp_func *` of the found function after resolving -/// `fully_qualified_function_name` if found. If there is no such function -/// std::nullopt is returned. `memory` is used for temporary allocations -/// inside this function. ModulePtr must be kept alive to make sure it won't be unloaded. -std::optional> FindFunction( - const ModuleRegistry &module_registry, std::string_view fully_qualified_function_name, - utils::MemoryResource *memory); - -template -concept IsCallable = utils::SameAsAnyOf; - -template -void ConstructArguments(const std::vector &args, const TCall &callable, - const std::string_view fully_qualified_name, mgp_list &args_list, mgp_graph &graph) { - const auto n_args = args.size(); - const auto c_args_sz = callable.args.size(); - const auto c_opt_args_sz = callable.opt_args.size(); - - if (n_args < c_args_sz || (n_args - c_args_sz > c_opt_args_sz)) { - if (callable.args.empty() && callable.opt_args.empty()) { - throw QueryRuntimeException("'{}' requires no arguments.", fully_qualified_name); - } - - if (callable.opt_args.empty()) { - throw QueryRuntimeException("'{}' requires exactly {} {}.", fully_qualified_name, c_args_sz, - c_args_sz == 1U ? "argument" : "arguments"); - } - - throw QueryRuntimeException("'{}' requires between {} and {} arguments.", fully_qualified_name, c_args_sz, - c_args_sz + c_opt_args_sz); - } - args_list.elems.reserve(n_args); - - auto is_not_optional_arg = [c_args_sz](int i) { return c_args_sz > i; }; - for (size_t i = 0; i < n_args; ++i) { - auto arg = args[i]; - std::string_view name; - const query::v2::procedure::CypherType *type; - if (is_not_optional_arg(i)) { - name = callable.args[i].first; - type = callable.args[i].second; - } else { - name = std::get<0>(callable.opt_args[i - c_args_sz]); - type = std::get<1>(callable.opt_args[i - c_args_sz]); - } - if (!type->SatisfiesType(arg)) { - throw QueryRuntimeException("'{}' argument named '{}' at position {} must be of type {}.", fully_qualified_name, - name, i, type->GetPresentableName()); - } - args_list.elems.emplace_back(std::move(arg), &graph); - } - // Fill missing optional arguments with their default values. - const size_t passed_in_opt_args = n_args - c_args_sz; - for (size_t i = passed_in_opt_args; i < c_opt_args_sz; ++i) { - args_list.elems.emplace_back(std::get<2>(callable.opt_args[i]), &graph); - } -} -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/py_module.cpp b/src/query/v2/procedure/py_module.cpp deleted file mode 100644 index 3295f6cff..000000000 --- a/src/query/v2/procedure/py_module.cpp +++ /dev/null @@ -1,2650 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/procedure/py_module.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include "mg_procedure.h" -#include "query/v2/procedure/mg_procedure_helpers.hpp" -#include "query/v2/procedure/mg_procedure_impl.hpp" -#include "utils/memory.hpp" -#include "utils/on_scope_exit.hpp" -#include "utils/pmr/vector.hpp" - -namespace memgraph::query::v2::procedure { - -namespace { -// Set this as a __reduce__ special method on our types to prevent `pickle` and -// `copy` module operations on our types. -PyObject *DisallowPickleAndCopy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - auto *type = Py_TYPE(self); - std::stringstream ss; - ss << "cannot pickle nor copy '" << type->tp_name << "' object"; - const auto &msg = ss.str(); - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return nullptr; -} - -PyObject *gMgpUnknownError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpUnableToAllocateError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpInsufficientBufferError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpOutOfRangeError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpLogicErrorError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpDeletedObjectError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpInvalidArgumentError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpKeyAlreadyExistsError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpImmutableObjectError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpValueConversionError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -PyObject *gMgpSerializationError{nullptr}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -// Returns true if an exception is raised -bool RaiseExceptionFromErrorCode(const mgp_error error) { - switch (error) { - case mgp_error::MGP_ERROR_NO_ERROR: - return false; - case mgp_error::MGP_ERROR_UNKNOWN_ERROR: { - PyErr_SetString(gMgpUnknownError, "Unknown error happened."); - return true; - } - case mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE: { - PyErr_SetString(gMgpUnableToAllocateError, "Unable to allocate memory."); - return true; - } - case mgp_error::MGP_ERROR_INSUFFICIENT_BUFFER: { - PyErr_SetString(gMgpInsufficientBufferError, "Insufficient buffer."); - return true; - } - case mgp_error::MGP_ERROR_OUT_OF_RANGE: { - PyErr_SetString(gMgpOutOfRangeError, "Out of range."); - return true; - } - case mgp_error::MGP_ERROR_LOGIC_ERROR: { - PyErr_SetString(gMgpLogicErrorError, "Logic error."); - return true; - } - case mgp_error::MGP_ERROR_DELETED_OBJECT: { - PyErr_SetString(gMgpDeletedObjectError, "Accessing deleted object."); - return true; - } - case mgp_error::MGP_ERROR_INVALID_ARGUMENT: { - PyErr_SetString(gMgpInvalidArgumentError, "Invalid argument."); - return true; - } - case mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS: { - PyErr_SetString(gMgpKeyAlreadyExistsError, "Key already exists."); - return true; - } - case mgp_error::MGP_ERROR_IMMUTABLE_OBJECT: { - PyErr_SetString(gMgpImmutableObjectError, "Cannot modify immutable object."); - return true; - } - case mgp_error::MGP_ERROR_VALUE_CONVERSION: { - PyErr_SetString(gMgpValueConversionError, "Value conversion failed."); - return true; - } - case mgp_error::MGP_ERROR_SERIALIZATION_ERROR: { - PyErr_SetString(gMgpSerializationError, "Operation cannot be serialized."); - return true; - } - } -} - -mgp_value *PyObjectToMgpValueWithPythonExceptions(PyObject *py_value, mgp_memory *memory) noexcept { - try { - return PyObjectToMgpValue(py_value, memory); - } catch (const std::bad_alloc &e) { - PyErr_SetString(PyExc_MemoryError, e.what()); - return nullptr; - } catch (const std::overflow_error &e) { - PyErr_SetString(PyExc_OverflowError, e.what()); - return nullptr; - } catch (const std::invalid_argument &e) { - PyErr_SetString(PyExc_ValueError, e.what()); - return nullptr; - } catch (const std::exception &e) { - PyErr_SetString(PyExc_RuntimeError, e.what()); - return nullptr; - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Unknown error happened"); - return nullptr; - } -} -} // namespace - -// Definitions of types wrapping C API types -// -// These should all be in the private `_mgp` Python module, which will be used -// by the `mgp` to implement the user friendly Python API. - -// Wraps mgp_graph in a PyObject. -// -// Executing a `CALL python_module.procedure(...)` in openCypher should -// instantiate exactly 1 mgp_graph instance. We will rely on this assumption in -// order to test for validity of usage. The idea is to clear the `graph` to -// `nullptr` after the execution completes. If a user stored a reference to -// `_mgp.Graph` in their global Python state, then we are no longer working with -// a valid graph so `nullptr` will catch this. `_mgp.Graph` provides `is_valid` -// method for checking this by our higher level API in `mgp` module. Python only -// does shallow copies by default, and we do not provide deep copy of -// `_mgp.Graph`, so this validity concept should work fine. -// -// clang-format off -struct PyGraph { - PyObject_HEAD - mgp_graph *graph; - mgp_memory *memory; -}; -// clang-format on - -// clang-format off -struct PyVerticesIterator { - PyObject_HEAD - mgp_vertices_iterator *it; - PyGraph *py_graph; -}; -// clang-format on - -PyObject *MakePyVertex(mgp_vertex &vertex, PyGraph *py_graph); - -void PyVerticesIteratorDealloc(PyVerticesIterator *self) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_vertices_iterator_destroy` if we are not in valid - // execution context. The query execution should free all memory used during - // execution, so we may cause a double free issue. - if (self->py_graph->graph) mgp_vertices_iterator_destroy(self->it); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyVerticesIteratorGet(PyVerticesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_vertex *vertex{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertices_iterator_get(self->it, &vertex))) { - return nullptr; - } - if (vertex == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - return MakePyVertex(*vertex, self->py_graph); -} - -PyObject *PyVerticesIteratorNext(PyVerticesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_vertex *vertex{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertices_iterator_next(self->it, &vertex))) { - return nullptr; - } - if (vertex == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - return MakePyVertex(*vertex, self->py_graph); -} - -static PyMethodDef PyVerticesIteratorMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"get", reinterpret_cast(PyVerticesIteratorGet), METH_NOARGS, - "Get the current vertex pointed to by the iterator or return None."}, - {"next", reinterpret_cast(PyVerticesIteratorNext), METH_NOARGS, - "Advance the iterator to the next vertex and return it."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyVerticesIteratorType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.VerticesIterator", - .tp_basicsize = sizeof(PyVerticesIterator), - .tp_dealloc = reinterpret_cast(PyVerticesIteratorDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_vertices_iterator.", - .tp_methods = PyVerticesIteratorMethods, -}; -// clang-format on - -// clang-format off -struct PyEdgesIterator { - PyObject_HEAD - mgp_edges_iterator *it; - PyGraph *py_graph; -}; -// clang-format on - -PyObject *MakePyEdge(mgp_edge &edge, PyGraph *py_graph); - -void PyEdgesIteratorDealloc(PyEdgesIterator *self) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_edges_iterator_destroy` if we are not in valid - // execution context. The query execution should free all memory used during - // execution, so we may cause a double free issue. - if (self->py_graph->graph) mgp_edges_iterator_destroy(self->it); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyEdgesIteratorGet(PyEdgesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edge *edge{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_edges_iterator_get(self->it, &edge))) { - return nullptr; - } - if (edge == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - return MakePyEdge(*edge, self->py_graph); -} - -PyObject *PyEdgesIteratorNext(PyEdgesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edge *edge{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_edges_iterator_next(self->it, &edge))) { - return nullptr; - } - if (edge == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - return MakePyEdge(*edge, self->py_graph); -} - -static PyMethodDef PyEdgesIteratorMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"get", reinterpret_cast(PyEdgesIteratorGet), METH_NOARGS, - "Get the current edge pointed to by the iterator or return None."}, - {"next", reinterpret_cast(PyEdgesIteratorNext), METH_NOARGS, - "Advance the iterator to the next edge and return it."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyEdgesIteratorType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.EdgesIterator", - .tp_basicsize = sizeof(PyEdgesIterator), - .tp_dealloc = reinterpret_cast(PyEdgesIteratorDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_edges_iterator.", - .tp_methods = PyEdgesIteratorMethods, -}; -// clang-format on - -PyObject *PyGraphInvalidate(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - self->graph = nullptr; - self->memory = nullptr; - Py_RETURN_NONE; -} - -bool PyGraphIsValidImpl(PyGraph &self) { return self.graph != nullptr; } - -PyObject *PyGraphIsValid(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(PyGraphIsValidImpl(*self)); -} - -PyObject *PyGraphIsMutable(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(CallBool(mgp_graph_is_mutable, self->graph)); -} - -PyObject *MakePyVertexWithoutCopy(mgp_vertex &vertex, PyGraph *py_graph); - -PyObject *PyGraphGetVertexById(PyGraph *self, PyObject *args) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - static_assert(std::is_same_v); - int64_t id = 0; - if (!PyArg_ParseTuple(args, "l", &id)) return nullptr; - mgp_vertex *vertex{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_graph_get_vertex_by_id(self->graph, mgp_vertex_id{id}, self->memory, &vertex))) { - return nullptr; - } - if (!vertex) { - PyErr_SetString(PyExc_IndexError, "Unable to find the vertex with given ID."); - return nullptr; - } - auto *py_vertex = MakePyVertexWithoutCopy(*vertex, self); - if (!py_vertex) mgp_vertex_destroy(vertex); - return py_vertex; -} - -PyObject *PyGraphCreateVertex(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - MgpUniquePtr new_vertex{nullptr, mgp_vertex_destroy}; - // TODO(jbajic) Fix query module - // if (RaiseExceptionFromErrorCode(CreateMgpObject(new_vertex, mgp_graph_create_vertex, self->graph, self->memory))) { - // return nullptr; - // } - auto *py_vertex = MakePyVertexWithoutCopy(*new_vertex, self); - if (py_vertex != nullptr) { - static_cast(new_vertex.release()); - } - return py_vertex; -} - -PyObject *PyGraphCreateEdge(PyGraph *self, PyObject *args); - -PyObject *PyGraphDeleteVertex(PyGraph *self, PyObject *args); - -PyObject *PyGraphDetachDeleteVertex(PyGraph *self, PyObject *args); - -PyObject *PyGraphDeleteEdge(PyGraph *self, PyObject *args); - -PyObject *PyGraphIterVertices(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - mgp_vertices_iterator *vertices_it{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_graph_iter_vertices(self->graph, self->memory, &vertices_it))) { - return nullptr; - } - auto *py_vertices_it = PyObject_New(PyVerticesIterator, &PyVerticesIteratorType); - if (!py_vertices_it) { - mgp_vertices_iterator_destroy(vertices_it); - return nullptr; - } - py_vertices_it->it = vertices_it; - Py_INCREF(self); - py_vertices_it->py_graph = self; - return reinterpret_cast(py_vertices_it); -} - -PyObject *PyGraphMustAbort(PyGraph *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - return PyBool_FromLong(mgp_must_abort(self->graph)); -} - -static PyMethodDef PyGraphMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"invalidate", reinterpret_cast(PyGraphInvalidate), METH_NOARGS, - "Invalidate the Graph context thus preventing the Graph from being used."}, - {"is_valid", reinterpret_cast(PyGraphIsValid), METH_NOARGS, - "Return True if Graph is in valid context and may be used."}, - {"is_mutable", reinterpret_cast(PyGraphIsMutable), METH_NOARGS, - "Return True if Graph is mutable and can be used to modify vertices and edges."}, - {"get_vertex_by_id", reinterpret_cast(PyGraphGetVertexById), METH_VARARGS, - "Get the vertex or raise IndexError."}, - {"create_vertex", reinterpret_cast(PyGraphCreateVertex), METH_NOARGS, "Create a vertex."}, - {"create_edge", reinterpret_cast(PyGraphCreateEdge), METH_VARARGS, "Create an edge."}, - {"delete_vertex", reinterpret_cast(PyGraphDeleteVertex), METH_VARARGS, "Delete a vertex."}, - {"detach_delete_vertex", reinterpret_cast(PyGraphDetachDeleteVertex), METH_VARARGS, - "Delete a vertex and all of its edges."}, - {"delete_edge", reinterpret_cast(PyGraphDeleteEdge), METH_VARARGS, "Delete an edge."}, - {"iter_vertices", reinterpret_cast(PyGraphIterVertices), METH_NOARGS, "Return _mgp.VerticesIterator."}, - {"must_abort", reinterpret_cast(PyGraphMustAbort), METH_NOARGS, - "Check whether the running procedure should abort"}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyGraphType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Graph", - .tp_basicsize = sizeof(PyGraph), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_graph.", - .tp_methods = PyGraphMethods, -}; -// clang-format on - -PyObject *MakePyGraph(mgp_graph *graph, mgp_memory *memory) { - MG_ASSERT(!graph || (graph && memory)); - auto *py_graph = PyObject_New(PyGraph, &PyGraphType); - if (!py_graph) return nullptr; - py_graph->graph = graph; - py_graph->memory = memory; - return reinterpret_cast(py_graph); -} - -// clang-format off -struct PyCypherType { - PyObject_HEAD - mgp_type *type; -}; -// clang-format on - -// clang-format off -static PyTypeObject PyCypherTypeType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Type", - .tp_basicsize = sizeof(PyCypherType), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_type.", -}; -// clang-format on - -PyObject *MakePyCypherType(mgp_type *type) { - MG_ASSERT(type); - auto *py_type = PyObject_New(PyCypherType, &PyCypherTypeType); - if (!py_type) return nullptr; - py_type->type = type; - return reinterpret_cast(py_type); -} - -// clang-format off -struct PyQueryProc { - PyObject_HEAD - mgp_proc *callable; -}; -// clang-format on - -// clang-format off -struct PyMagicFunc{ - PyObject_HEAD - mgp_func *callable; -}; -// clang-format on - -template -concept IsCallable = utils::SameAsAnyOf; - -template -PyObject *PyCallableAddArg(TCall *self, PyObject *args) { - MG_ASSERT(self->callable); - const char *name = nullptr; - PyCypherType *py_type = nullptr; - if (!PyArg_ParseTuple(args, "sO!", &name, &PyCypherTypeType, &py_type)) return nullptr; - auto *type = py_type->type; - - if constexpr (std::is_same_v) { - if (RaiseExceptionFromErrorCode(mgp_proc_add_arg(self->callable, name, type))) { - return nullptr; - } - } else if constexpr (std::is_same_v) { - if (RaiseExceptionFromErrorCode(mgp_func_add_arg(self->callable, name, type))) { - return nullptr; - } - } - - Py_RETURN_NONE; -} - -template -PyObject *PyCallableAddOptArg(TCall *self, PyObject *args) { - MG_ASSERT(self->callable); - const char *name = nullptr; - PyCypherType *py_type = nullptr; - PyObject *py_value = nullptr; - if (!PyArg_ParseTuple(args, "sO!O", &name, &PyCypherTypeType, &py_type, &py_value)) return nullptr; - auto *type = py_type->type; - mgp_memory memory{self->callable->opt_args.get_allocator().GetMemoryResource()}; - mgp_value *value = PyObjectToMgpValueWithPythonExceptions(py_value, &memory); - if (value == nullptr) { - return nullptr; - } - if constexpr (std::is_same_v) { - if (RaiseExceptionFromErrorCode(mgp_proc_add_opt_arg(self->callable, name, type, value))) { - mgp_value_destroy(value); - return nullptr; - } - } else if constexpr (std::is_same_v) { - if (RaiseExceptionFromErrorCode(mgp_func_add_opt_arg(self->callable, name, type, value))) { - mgp_value_destroy(value); - return nullptr; - } - } - - mgp_value_destroy(value); - Py_RETURN_NONE; -} - -PyObject *PyQueryProcAddArg(PyQueryProc *self, PyObject *args) { return PyCallableAddArg(self, args); } - -PyObject *PyQueryProcAddOptArg(PyQueryProc *self, PyObject *args) { return PyCallableAddOptArg(self, args); } - -PyObject *PyQueryProcAddResult(PyQueryProc *self, PyObject *args) { - MG_ASSERT(self->callable); - const char *name = nullptr; - PyCypherType *py_type = nullptr; - if (!PyArg_ParseTuple(args, "sO!", &name, &PyCypherTypeType, &py_type)) return nullptr; - - auto *type = reinterpret_cast(py_type)->type; - if (RaiseExceptionFromErrorCode(mgp_proc_add_result(self->callable, name, type))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyQueryProcAddDeprecatedResult(PyQueryProc *self, PyObject *args) { - MG_ASSERT(self->callable); - const char *name = nullptr; - PyCypherType *py_type = nullptr; - if (!PyArg_ParseTuple(args, "sO!", &name, &PyCypherTypeType, &py_type)) return nullptr; - auto *type = reinterpret_cast(py_type)->type; - if (RaiseExceptionFromErrorCode(mgp_proc_add_deprecated_result(self->callable, name, type))) { - return nullptr; - } - Py_RETURN_NONE; -} - -static PyMethodDef PyQueryProcMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"add_arg", reinterpret_cast(PyQueryProcAddArg), METH_VARARGS, - "Add a required argument to a procedure."}, - {"add_opt_arg", reinterpret_cast(PyQueryProcAddOptArg), METH_VARARGS, - "Add an optional argument with a default value to a procedure."}, - {"add_result", reinterpret_cast(PyQueryProcAddResult), METH_VARARGS, - "Add a result field to a procedure."}, - {"add_deprecated_result", reinterpret_cast(PyQueryProcAddDeprecatedResult), METH_VARARGS, - "Add a result field to a procedure and mark it as deprecated."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyQueryProcType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Proc", - .tp_basicsize = sizeof(PyQueryProc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_proc.", - .tp_methods = PyQueryProcMethods, -}; -// clang-format on - -PyObject *PyMagicFuncAddArg(PyMagicFunc *self, PyObject *args) { return PyCallableAddArg(self, args); } - -PyObject *PyMagicFuncAddOptArg(PyMagicFunc *self, PyObject *args) { return PyCallableAddOptArg(self, args); } - -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -static PyMethodDef PyMagicFuncMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"add_arg", reinterpret_cast(PyMagicFuncAddArg), METH_VARARGS, - "Add a required argument to a function."}, - {"add_opt_arg", reinterpret_cast(PyMagicFuncAddOptArg), METH_VARARGS, - "Add an optional argument with a default value to a function."}, - {nullptr}, -}; - -// clang-format off -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -static PyTypeObject PyMagicFuncType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Func", - .tp_basicsize = sizeof(PyMagicFunc), - // NOLINTNEXTLINE(hicpp-signed-bitwise) - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_func.", - .tp_methods = PyMagicFuncMethods, -}; -// clang-format on - -// clang-format off -struct PyQueryModule { - PyObject_HEAD - mgp_module *module; -}; -// clang-format on - -struct PyMessages { - PyObject_HEAD; - mgp_messages *messages; - mgp_memory *memory; -}; - -struct PyMessage { - PyObject_HEAD; - mgp_message *message; - const PyMessages *messages; - mgp_memory *memory; -}; - -PyObject *PyMessagesIsValid(const PyMessages *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(!!self->messages); -} - -PyObject *PyMessageIsValid(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - return PyMessagesIsValid(self->messages, nullptr); -} - -PyObject *PyMessageGetSourceType(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - MG_ASSERT(self->memory); - mgp_source_type source_type{mgp_source_type::KAFKA}; - if (RaiseExceptionFromErrorCode(mgp_message_source_type(self->message, &source_type))) { - return nullptr; - } - auto *py_source_type = PyLong_FromLong(static_cast(source_type)); - if (!py_source_type) { - PyErr_SetString(PyExc_RuntimeError, "Unable to get long from source type"); - return nullptr; - } - return py_source_type; -} - -PyObject *PyMessageGetPayload(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - size_t payload_size{0}; - if (RaiseExceptionFromErrorCode(mgp_message_payload_size(self->message, &payload_size))) { - return nullptr; - } - const char *payload{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_message_payload(self->message, &payload))) { - return nullptr; - } - auto *raw_bytes = PyByteArray_FromStringAndSize(payload, payload_size); - if (!raw_bytes) { - PyErr_SetString(PyExc_RuntimeError, "Unable to get raw bytes from payload"); - return nullptr; - } - return raw_bytes; -} - -PyObject *PyMessageGetTopicName(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - MG_ASSERT(self->memory); - const char *topic_name{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_message_topic_name(self->message, &topic_name))) { - return nullptr; - } - auto *py_topic_name = PyUnicode_FromString(topic_name); - if (!py_topic_name) { - PyErr_SetString(PyExc_RuntimeError, "Unable to get string from topic_name"); - return nullptr; - } - return py_topic_name; -} - -PyObject *PyMessageGetKey(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - MG_ASSERT(self->memory); - size_t key_size{0}; - if (RaiseExceptionFromErrorCode(mgp_message_key_size(self->message, &key_size))) { - return nullptr; - } - const char *key{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_message_key(self->message, &key))) { - return nullptr; - } - auto *raw_bytes = PyByteArray_FromStringAndSize(key, key_size); - if (!raw_bytes) { - PyErr_SetString(PyExc_RuntimeError, "Unable to get raw bytes from payload"); - return nullptr; - } - return raw_bytes; -} - -PyObject *PyMessageGetTimestamp(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - MG_ASSERT(self->memory); - int64_t timestamp{0}; - if (RaiseExceptionFromErrorCode(mgp_message_timestamp(self->message, ×tamp))) { - return nullptr; - } - auto *py_int = PyLong_FromUnsignedLong(timestamp); - if (!py_int) { - PyErr_SetString(PyExc_IndexError, "Unable to get timestamp."); - return nullptr; - } - return py_int; -} - -PyObject *PyMessageGetOffset(PyMessage *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->message); - MG_ASSERT(self->memory); - int64_t offset{0}; - if (RaiseExceptionFromErrorCode(mgp_message_offset(self->message, &offset))) { - return nullptr; - } - auto *py_int = PyLong_FromLongLong(offset); - if (!py_int) { - PyErr_SetString(PyExc_IndexError, "Unable to get offset"); - return nullptr; - } - return py_int; -} - -// NOLINTNEXTLINE -static PyMethodDef PyMessageMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"is_valid", reinterpret_cast(PyMessageIsValid), METH_NOARGS, - "Return True if messages is in valid context and may be used."}, - {"source_type", reinterpret_cast(PyMessageGetSourceType), METH_NOARGS, "Get stream source type."}, - {"payload", reinterpret_cast(PyMessageGetPayload), METH_NOARGS, "Get payload"}, - {"topic_name", reinterpret_cast(PyMessageGetTopicName), METH_NOARGS, "Get topic name."}, - {"key", reinterpret_cast(PyMessageGetKey), METH_NOARGS, "Get message key."}, - {"timestamp", reinterpret_cast(PyMessageGetTimestamp), METH_NOARGS, "Get message timestamp."}, - {"offset", reinterpret_cast(PyMessageGetOffset), METH_NOARGS, "Get message offset."}, - {nullptr}, -}; - -void PyMessageDealloc(PyMessage *self) { - MG_ASSERT(self->memory); - MG_ASSERT(self->message); - MG_ASSERT(self->messages); - // NOLINTNEXTLINE - Py_DECREF(self->messages); - // NOLINTNEXTLINE - Py_TYPE(self)->tp_free(self); -} - -// NOLINTNEXTLINE -static PyTypeObject PyMessageType = { - PyVarObject_HEAD_INIT(nullptr, 0).tp_name = "_mgp.Message", - .tp_basicsize = sizeof(PyMessage), - .tp_dealloc = reinterpret_cast(PyMessageDealloc), - // NOLINTNEXTLINE - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_message.", - // NOLINTNEXTLINE - .tp_methods = PyMessageMethods, -}; - -PyObject *PyMessagesInvalidate(PyMessages *self, PyObject *Py_UNUSED(ignored)) { - self->messages = nullptr; - self->memory = nullptr; - Py_RETURN_NONE; -} - -PyObject *PyMessagesGetTotalMessages(PyMessages *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->messages); - MG_ASSERT(self->memory); - auto size = self->messages->messages.size(); - auto *py_int = PyLong_FromSize_t(size); - if (!py_int) { - PyErr_SetString(PyExc_IndexError, "Unable to get total messages count."); - return nullptr; - } - return py_int; -} - -PyObject *PyMessagesGetMessageAt(PyMessages *self, PyObject *args) { - MG_ASSERT(self->messages); - MG_ASSERT(self->memory); - int64_t id = 0; - if (!PyArg_ParseTuple(args, "l", &id)) return nullptr; - if (id < 0 || id >= self->messages->messages.size()) return nullptr; - auto *message = &self->messages->messages[id]; - // NOLINTNEXTLINE - auto *py_message = PyObject_New(PyMessage, &PyMessageType); - if (!py_message) { - return nullptr; - } - py_message->message = message; - // NOLINTNEXTLINE - Py_INCREF(self); - py_message->messages = self; - py_message->memory = self->memory; - if (!message) { - PyErr_SetString(PyExc_IndexError, "Unable to find the message with given index."); - return nullptr; - } - // NOLINTNEXTLINE - return reinterpret_cast(py_message); -} - -// NOLINTNEXTLINE -static PyMethodDef PyMessagesMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"invalidate", reinterpret_cast(PyMessagesInvalidate), METH_NOARGS, - "Invalidate the messages context thus preventing the messages from being used"}, - {"is_valid", reinterpret_cast(PyMessagesIsValid), METH_NOARGS, - "Return True if messages is in valid context and may be used."}, - {"total_messages", reinterpret_cast(PyMessagesGetTotalMessages), METH_VARARGS, - "Get number of messages available"}, - {"message_at", reinterpret_cast(PyMessagesGetMessageAt), METH_VARARGS, - "Get message at index idx from messages"}, - {nullptr}, -}; - -// NOLINTNEXTLINE -static PyTypeObject PyMessagesType = { - PyVarObject_HEAD_INIT(nullptr, 0).tp_name = "_mgp.Messages", - .tp_basicsize = sizeof(PyMessages), - // NOLINTNEXTLINE - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_messages.", - // NOLINTNEXTLINE - .tp_methods = PyMessagesMethods, -}; - -PyObject *MakePyMessages(mgp_messages *msgs, mgp_memory *memory) { - MG_ASSERT(!msgs || (msgs && memory)); - // NOLINTNEXTLINE - auto *py_messages = PyObject_New(PyMessages, &PyMessagesType); - if (!py_messages) return nullptr; - py_messages->messages = msgs; - py_messages->memory = memory; - return reinterpret_cast(py_messages); -} - -py::Object MgpListToPyTuple(mgp_list *list, PyGraph *py_graph) { - MG_ASSERT(list); - MG_ASSERT(py_graph); - const auto len = list->elems.size(); - py::Object py_tuple(PyTuple_New(len)); - if (!py_tuple) return nullptr; - for (size_t i = 0; i < len; ++i) { - auto elem = MgpValueToPyObject(list->elems[i], py_graph); - if (!elem) return nullptr; - // Explicitly convert `py_tuple`, which is `py::Object`, via static_cast. - // Then the macro will cast it to `PyTuple *`. - PyTuple_SET_ITEM(py_tuple.Ptr(), i, elem.Steal()); - } - return py_tuple; -} - -py::Object MgpListToPyTuple(mgp_list *list, PyObject *py_graph) { - if (Py_TYPE(py_graph) != &PyGraphType) { - PyErr_SetString(PyExc_TypeError, "Expected a _mgp.Graph."); - return nullptr; - } - return MgpListToPyTuple(list, reinterpret_cast(py_graph)); -} - -namespace { -std::optional AddRecordFromPython(mgp_result *result, py::Object py_record) { - py::Object py_mgp(PyImport_ImportModule("mgp")); - if (!py_mgp) return py::FetchError(); - auto record_cls = py_mgp.GetAttr("Record"); - if (!record_cls) return py::FetchError(); - if (!PyObject_IsInstance(py_record.Ptr(), record_cls.Ptr())) { - std::stringstream ss; - ss << "Value '" << py_record << "' is not an instance of 'mgp.Record'"; - const auto &msg = ss.str(); - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return py::FetchError(); - } - py::Object fields(py_record.GetAttr("fields")); - if (!fields) return py::FetchError(); - if (!PyDict_Check(fields)) { - PyErr_SetString(PyExc_TypeError, "Expected 'mgp.Record.fields' to be a 'dict'"); - return py::FetchError(); - } - py::Object items(PyDict_Items(fields.Ptr())); - if (!items) return py::FetchError(); - mgp_result_record *record{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_result_new_record(result, &record))) { - return py::FetchError(); - } - Py_ssize_t len = PyList_GET_SIZE(items.Ptr()); - for (Py_ssize_t i = 0; i < len; ++i) { - auto *item = PyList_GET_ITEM(items.Ptr(), i); - if (!item) return py::FetchError(); - MG_ASSERT(PyTuple_Check(item)); - auto *key = PyTuple_GetItem(item, 0); - if (!key) return py::FetchError(); - if (!PyUnicode_Check(key)) { - std::stringstream ss; - ss << "Field name '" << py::Object::FromBorrow(key) << "' is not an instance of 'str'"; - const auto &msg = ss.str(); - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return py::FetchError(); - } - const auto *field_name = PyUnicode_AsUTF8(key); - if (!field_name) return py::FetchError(); - auto *val = PyTuple_GetItem(item, 1); - if (!val) return py::FetchError(); - mgp_memory memory{result->rows.get_allocator().GetMemoryResource()}; - mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, &memory); - if (field_val == nullptr) { - return py::FetchError(); - } - if (mgp_result_record_insert(record, field_name, field_val) != mgp_error::MGP_ERROR_NO_ERROR) { - std::stringstream ss; - ss << "Unable to insert field '" << py::Object::FromBorrow(key) << "' with value: '" - << py::Object::FromBorrow(val) << "'; did you set the correct field type?"; - const auto &msg = ss.str(); - PyErr_SetString(PyExc_ValueError, msg.c_str()); - mgp_value_destroy(field_val); - return py::FetchError(); - } - mgp_value_destroy(field_val); - } - return std::nullopt; -} - -std::optional AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq) { - Py_ssize_t len = PySequence_Size(py_seq.Ptr()); - if (len == -1) return py::FetchError(); - for (Py_ssize_t i = 0; i < len; ++i) { - py::Object py_record(PySequence_GetItem(py_seq.Ptr(), i)); - if (!py_record) return py::FetchError(); - auto maybe_exc = AddRecordFromPython(result, py_record); - if (maybe_exc) return maybe_exc; - } - return std::nullopt; -} - -std::function PyObjectCleanup(py::Object &py_object) { - return [py_object]() { - // Run `gc.collect` (reference cycle-detection) explicitly, so that we are - // sure the procedure cleaned up everything it held references to. If the - // user stored a reference to one of our `_mgp` instances then the - // internally used `mgp_*` structs will stay unfreed and a memory leak - // will be reported at the end of the query execution. - py::Object gc(PyImport_ImportModule("gc")); - if (!gc) { - LOG_FATAL(py::FetchError().value()); - } - - if (!gc.CallMethod("collect")) { - LOG_FATAL(py::FetchError().value()); - } - - // After making sure all references from our side have been cleared, - // invalidate the `_mgp.Graph` object. If the user kept a reference to one - // of our `_mgp` instances then this will prevent them from using those - // objects (whose internal `mgp_*` pointers are now invalid and would cause - // a crash). - if (!py_object.CallMethod("invalidate")) { - LOG_FATAL(py::FetchError().value()); - } - }; -} - -void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *graph, mgp_result *result, - mgp_memory *memory) { - auto gil = py::EnsureGIL(); - - auto error_to_msg = [](const std::optional &exc_info) -> std::optional { - if (!exc_info) return std::nullopt; - // Here we tell the traceback formatter to skip the first line of the - // traceback because that line will always be our wrapper function in our - // internal `mgp.py` file. With that line skipped, the user will always - // get only the relevant traceback that happened in his Python code. - return py::FormatException(*exc_info, /* skip_first_line = */ true); - }; - - auto call = [&](py::Object py_graph) -> std::optional { - py::Object py_args(MgpListToPyTuple(args, py_graph.Ptr())); - if (!py_args) return py::FetchError(); - auto py_res = py_cb.Call(py_graph, py_args); - if (!py_res) return py::FetchError(); - if (PySequence_Check(py_res.Ptr())) { - return AddMultipleRecordsFromPython(result, py_res); - } else { - return AddRecordFromPython(result, py_res); - } - }; - - // It is *VERY IMPORTANT* to note that this code takes great care not to keep - // any extra references to any `_mgp` instances (except for `_mgp.Graph`), so - // as not to introduce extra reference counts and prevent their deallocation. - // In particular, the `ExceptionInfo` object has a `traceback` field that - // contains references to the Python frames and their arguments, and therefore - // our `_mgp` instances as well. Within this code we ensure not to keep the - // `ExceptionInfo` object alive so that no extra reference counts are - // introduced. We only fetch the error message and immediately destroy the - // object. - std::optional maybe_msg; - { - py::Object py_graph(MakePyGraph(graph, memory)); - utils::OnScopeExit clean_up(PyObjectCleanup(py_graph)); - if (py_graph) { - maybe_msg = error_to_msg(call(py_graph)); - } else { - maybe_msg = error_to_msg(py::FetchError()); - } - } - - if (maybe_msg) { - static_cast(mgp_result_set_error_msg(result, maybe_msg->c_str())); - } -} - -void CallPythonTransformation(const py::Object &py_cb, mgp_messages *msgs, mgp_graph *graph, mgp_result *result, - mgp_memory *memory) { - auto gil = py::EnsureGIL(); - - auto error_to_msg = [](const std::optional &exc_info) -> std::optional { - if (!exc_info) return std::nullopt; - // Here we tell the traceback formatter to skip the first line of the - // traceback because that line will always be our wrapper function in our - // internal `mgp.py` file. With that line skipped, the user will always - // get only the relevant traceback that happened in his Python code. - return py::FormatException(*exc_info, /* skip_first_line = */ true); - }; - - auto call = [&](py::Object py_graph, py::Object py_messages) -> std::optional { - auto py_res = py_cb.Call(py_graph, py_messages); - if (!py_res) return py::FetchError(); - if (PySequence_Check(py_res.Ptr())) { - return AddMultipleRecordsFromPython(result, py_res); - } - return AddRecordFromPython(result, py_res); - }; - - // It is *VERY IMPORTANT* to note that this code takes great care not to keep - // any extra references to any `_mgp` instances (except for `_mgp.Graph`), so - // as not to introduce extra reference counts and prevent their deallocation. - // In particular, the `ExceptionInfo` object has a `traceback` field that - // contains references to the Python frames and their arguments, and therefore - // our `_mgp` instances as well. Within this code we ensure not to keep the - // `ExceptionInfo` object alive so that no extra reference counts are - // introduced. We only fetch the error message and immediately destroy the - // object. - std::optional maybe_msg; - { - py::Object py_graph(MakePyGraph(graph, memory)); - py::Object py_messages(MakePyMessages(msgs, memory)); - - utils::OnScopeExit clean_up_graph(PyObjectCleanup(py_graph)); - utils::OnScopeExit clean_up_messages(PyObjectCleanup(py_messages)); - - if (py_graph && py_messages) { - maybe_msg = error_to_msg(call(py_graph, py_messages)); - } else { - maybe_msg = error_to_msg(py::FetchError()); - } - } - - if (maybe_msg) { - static_cast(mgp_result_set_error_msg(result, maybe_msg->c_str())); - } -} - -void CallPythonFunction(const py::Object &py_cb, mgp_list *args, mgp_graph *graph, mgp_func_result *result, - mgp_memory *memory) { - auto gil = py::EnsureGIL(); - - auto error_to_msg = [](const std::optional &exc_info) -> std::optional { - if (!exc_info) return std::nullopt; - // Here we tell the traceback formatter to skip the first line of the - // traceback because that line will always be our wrapper function in our - // internal `mgp.py` file. With that line skipped, the user will always - // get only the relevant traceback that happened in his Python code. - return py::FormatException(*exc_info, /* skip_first_line = */ true); - }; - - auto call = [&](py::Object py_graph) -> utils::BasicResult, mgp_value *> { - py::Object py_args(MgpListToPyTuple(args, py_graph.Ptr())); - if (!py_args) return {py::FetchError()}; - auto py_res = py_cb.Call(py_graph, py_args); - if (!py_res) return {py::FetchError()}; - mgp_value *ret_val = PyObjectToMgpValueWithPythonExceptions(py_res.Ptr(), memory); - if (ret_val == nullptr) { - return {py::FetchError()}; - } - return ret_val; - }; - - // It is *VERY IMPORTANT* to note that this code takes great care not to keep - // any extra references to any `_mgp` instances (except for `_mgp.Graph`), so - // as not to introduce extra reference counts and prevent their deallocation. - // In particular, the `ExceptionInfo` object has a `traceback` field that - // contains references to the Python frames and their arguments, and therefore - // our `_mgp` instances as well. Within this code we ensure not to keep the - // `ExceptionInfo` object alive so that no extra reference counts are - // introduced. We only fetch the error message and immediately destroy the - // object. - std::optional maybe_msg; - { - py::Object py_graph(MakePyGraph(graph, memory)); - utils::OnScopeExit clean_up(PyObjectCleanup(py_graph)); - if (py_graph) { - auto maybe_result = call(py_graph); - if (!maybe_result.HasError()) { - static_cast(mgp_func_result_set_value(result, maybe_result.GetValue(), memory)); - return; - } - maybe_msg = error_to_msg(maybe_result.GetError()); - } else { - maybe_msg = error_to_msg(py::FetchError()); - } - } - - if (maybe_msg) { - static_cast( - mgp_func_result_set_error_msg(result, maybe_msg->c_str(), memory)); // No error fetching if this fails - } -} - -PyObject *PyQueryModuleAddProcedure(PyQueryModule *self, PyObject *cb, bool is_write_procedure) { - MG_ASSERT(self->module); - if (!PyCallable_Check(cb)) { - PyErr_SetString(PyExc_TypeError, "Expected a callable object."); - return nullptr; - } - auto py_cb = py::Object::FromBorrow(cb); - py::Object py_name(py_cb.GetAttr("__name__")); - const auto *name = PyUnicode_AsUTF8(py_name.Ptr()); - if (!name) return nullptr; - if (!IsValidIdentifierName(name)) { - PyErr_SetString(PyExc_ValueError, "Procedure name is not a valid identifier"); - return nullptr; - } - auto *memory = self->module->procedures.get_allocator().GetMemoryResource(); - mgp_proc proc(name, - [py_cb](mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory *memory) { - CallPythonProcedure(py_cb, args, graph, result, memory); - }, - memory, {.is_write = is_write_procedure}); - const auto &[proc_it, did_insert] = self->module->procedures.emplace(name, std::move(proc)); - if (!did_insert) { - PyErr_SetString(PyExc_ValueError, "Already registered a procedure with the same name."); - return nullptr; - } - auto *py_proc = PyObject_New(PyQueryProc, &PyQueryProcType); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - if (!py_proc) return nullptr; - py_proc->callable = &proc_it->second; - return reinterpret_cast(py_proc); -} -} // namespace - -PyObject *PyQueryModuleAddReadProcedure(PyQueryModule *self, PyObject *cb) { - return PyQueryModuleAddProcedure(self, cb, false); -} - -PyObject *PyQueryModuleAddWriteProcedure(PyQueryModule *self, PyObject *cb) { - return PyQueryModuleAddProcedure(self, cb, true); -} - -PyObject *PyQueryModuleAddTransformation(PyQueryModule *self, PyObject *cb) { - MG_ASSERT(self->module); - if (!PyCallable_Check(cb)) { - PyErr_SetString(PyExc_TypeError, "Expected a callable object."); - return nullptr; - } - auto py_cb = py::Object::FromBorrow(cb); - py::Object py_name(py_cb.GetAttr("__name__")); - const auto *name = PyUnicode_AsUTF8(py_name.Ptr()); - if (!name) return nullptr; - if (!IsValidIdentifierName(name)) { - PyErr_SetString(PyExc_ValueError, "Transformation name is not a valid identifier"); - return nullptr; - } - auto *memory = self->module->transformations.get_allocator().GetMemoryResource(); - mgp_trans trans( - name, - [py_cb](mgp_messages *msgs, mgp_graph *graph, mgp_result *result, mgp_memory *memory) { - CallPythonTransformation(py_cb, msgs, graph, result, memory); - }, - memory); - const auto [trans_it, did_insert] = self->module->transformations.emplace(name, std::move(trans)); - if (!did_insert) { - PyErr_SetString(PyExc_ValueError, "Already registered a procedure with the same name."); - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyQueryModuleAddFunction(PyQueryModule *self, PyObject *cb) { - MG_ASSERT(self->module); - if (!PyCallable_Check(cb)) { - PyErr_SetString(PyExc_TypeError, "Expected a callable object."); - return nullptr; - } - auto py_cb = py::Object::FromBorrow(cb); - py::Object py_name(py_cb.GetAttr("__name__")); - const auto *name = PyUnicode_AsUTF8(py_name.Ptr()); - if (!name) return nullptr; - if (!IsValidIdentifierName(name)) { - PyErr_SetString(PyExc_ValueError, "Function name is not a valid identifier"); - return nullptr; - } - auto *memory = self->module->functions.get_allocator().GetMemoryResource(); - mgp_func func( - name, - [py_cb](mgp_list *args, mgp_func_context *func_ctx, mgp_func_result *result, mgp_memory *memory) { - auto graph = mgp_graph::NonWritableGraph(*(func_ctx->impl), func_ctx->view); - return CallPythonFunction(py_cb, args, &graph, result, memory); - }, - memory); - const auto [func_it, did_insert] = self->module->functions.emplace(name, std::move(func)); - if (!did_insert) { - PyErr_SetString(PyExc_ValueError, "Already registered a function with the same name."); - return nullptr; - } - auto *py_func = PyObject_New(PyMagicFunc, &PyMagicFuncType); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - if (!py_func) return nullptr; - py_func->callable = &func_it->second; - return reinterpret_cast(py_func); -} - -static PyMethodDef PyQueryModuleMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"add_read_procedure", reinterpret_cast(PyQueryModuleAddReadProcedure), METH_O, - "Register a read-only procedure with this module."}, - {"add_write_procedure", reinterpret_cast(PyQueryModuleAddWriteProcedure), METH_O, - "Register a writeable procedure with this module."}, - {"add_transformation", reinterpret_cast(PyQueryModuleAddTransformation), METH_O, - "Register a transformation with this module."}, - {"add_function", reinterpret_cast(PyQueryModuleAddFunction), METH_O, - "Register a function with this module."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyQueryModuleType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Module", - .tp_basicsize = sizeof(PyQueryModule), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_module.", - .tp_methods = PyQueryModuleMethods, -}; -// clang-format on - -PyObject *MakePyQueryModule(mgp_module *module) { - MG_ASSERT(module); - auto *py_query_module = PyObject_New(PyQueryModule, &PyQueryModuleType); - if (!py_query_module) return nullptr; - py_query_module->module = module; - return reinterpret_cast(py_query_module); -} - -PyObject *PyMgpModuleTypeNullable(PyObject *mod, PyObject *obj) { - if (Py_TYPE(obj) != &PyCypherTypeType) { - PyErr_SetString(PyExc_TypeError, "Expected a _mgp.Type."); - return nullptr; - } - auto *py_type = reinterpret_cast(obj); - mgp_type *type{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_type_nullable(py_type->type, &type))) { - return nullptr; - } - return MakePyCypherType(type); -} - -PyObject *PyMgpModuleTypeList(PyObject *mod, PyObject *obj) { - if (Py_TYPE(obj) != &PyCypherTypeType) { - PyErr_SetString(PyExc_TypeError, "Expected a _mgp.Type."); - return nullptr; - } - auto *py_type = reinterpret_cast(obj); - mgp_type *type{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_type_list(py_type->type, &type))) { - return nullptr; - } - return MakePyCypherType(type); -} - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define DEFINE_PY_MGP_MODULE_TYPE(capital_type, small_type) \ - PyObject *PyMgpModuleType##capital_type(PyObject * /*mod*/, PyObject *Py_UNUSED(ignored)) { \ - mgp_type *type{nullptr}; \ - if (RaiseExceptionFromErrorCode(mgp_type_##small_type(&type))) { \ - return nullptr; \ - } \ - return MakePyCypherType(type); \ - } - -DEFINE_PY_MGP_MODULE_TYPE(Any, any); -DEFINE_PY_MGP_MODULE_TYPE(Bool, bool); -DEFINE_PY_MGP_MODULE_TYPE(String, string); -DEFINE_PY_MGP_MODULE_TYPE(Int, int); -DEFINE_PY_MGP_MODULE_TYPE(Float, float); -DEFINE_PY_MGP_MODULE_TYPE(Number, number); -DEFINE_PY_MGP_MODULE_TYPE(Map, map); -DEFINE_PY_MGP_MODULE_TYPE(Node, node); -DEFINE_PY_MGP_MODULE_TYPE(Relationship, relationship); -DEFINE_PY_MGP_MODULE_TYPE(Path, path); -DEFINE_PY_MGP_MODULE_TYPE(Date, date); -DEFINE_PY_MGP_MODULE_TYPE(LocalTime, local_time); -DEFINE_PY_MGP_MODULE_TYPE(LocalDateTime, local_date_time); -DEFINE_PY_MGP_MODULE_TYPE(Duration, duration); - -static PyMethodDef PyMgpModuleMethods[] = { - {"type_nullable", PyMgpModuleTypeNullable, METH_O, - "Build a type representing either a `null` value or a value of given " - "type."}, - {"type_list", PyMgpModuleTypeList, METH_O, "Build a type representing a list of values of given type."}, - {"type_any", PyMgpModuleTypeAny, METH_NOARGS, "Get the type representing any value that isn't `null`."}, - {"type_bool", PyMgpModuleTypeBool, METH_NOARGS, "Get the type representing boolean values."}, - {"type_string", PyMgpModuleTypeString, METH_NOARGS, "Get the type representing string values."}, - {"type_int", PyMgpModuleTypeInt, METH_NOARGS, "Get the type representing integer values."}, - {"type_float", PyMgpModuleTypeFloat, METH_NOARGS, "Get the type representing floating-point values."}, - {"type_number", PyMgpModuleTypeNumber, METH_NOARGS, "Get the type representing any number value."}, - {"type_map", PyMgpModuleTypeMap, METH_NOARGS, "Get the type representing map values."}, - {"type_node", PyMgpModuleTypeNode, METH_NOARGS, "Get the type representing graph node values."}, - {"type_relationship", PyMgpModuleTypeRelationship, METH_NOARGS, - "Get the type representing graph relationship values."}, - {"type_path", PyMgpModuleTypePath, METH_NOARGS, - "Get the type representing a graph path (walk) from one node to another."}, - {"type_date", PyMgpModuleTypeDate, METH_NOARGS, "Get the type representing a Date."}, - {"type_local_time", PyMgpModuleTypeLocalTime, METH_NOARGS, "Get the type representing a LocalTime."}, - {"type_local_date_time", PyMgpModuleTypeLocalDateTime, METH_NOARGS, "Get the type representing a LocalDateTime."}, - {"type_duration", PyMgpModuleTypeDuration, METH_NOARGS, "Get the type representing a Duration."}, - {nullptr}, -}; - -// clang-format off -static PyModuleDef PyMgpModule = { - PyModuleDef_HEAD_INIT, - .m_name = "_mgp", - .m_doc = "Contains raw bindings to mg_procedure.h C API.", - .m_size = -1, - .m_methods = PyMgpModuleMethods, -}; -// clang-format on - -// clang-format off -struct PyPropertiesIterator { - PyObject_HEAD - mgp_properties_iterator *it; - PyGraph *py_graph; -}; -// clang-format on - -void PyPropertiesIteratorDealloc(PyPropertiesIterator *self) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_properties_iterator_destroy` if we are not in valid - // execution context. The query execution should free all memory used during - // execution, so we may cause a double free issue. - if (self->py_graph->graph) mgp_properties_iterator_destroy(self->it); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyPropertiesIteratorGet(PyPropertiesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_property *property{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_properties_iterator_get(self->it, &property))) { - return nullptr; - } - if (property == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - py::Object py_name(PyUnicode_FromString(property->name)); - if (!py_name) return nullptr; - auto py_value = MgpValueToPyObject(*property->value, self->py_graph); - if (!py_value) return nullptr; - return PyTuple_Pack(2, py_name.Ptr(), py_value.Ptr()); -} - -PyObject *PyPropertiesIteratorNext(PyPropertiesIterator *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->it); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_property *property{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_properties_iterator_next(self->it, &property))) { - return nullptr; - } - if (property == nullptr) { - Py_INCREF(Py_None); - return Py_None; - } - py::Object py_name(PyUnicode_FromString(property->name)); - if (!py_name) return nullptr; - auto py_value = MgpValueToPyObject(*property->value, self->py_graph); - if (!py_value) return nullptr; - return PyTuple_Pack(2, py_name.Ptr(), py_value.Ptr()); -} - -static PyMethodDef PyPropertiesIteratorMethods[] = { - {"get", reinterpret_cast(PyPropertiesIteratorGet), METH_NOARGS, - "Get the current proprety pointed to by the iterator or return None."}, - {"next", reinterpret_cast(PyPropertiesIteratorNext), METH_NOARGS, - "Advance the iterator to the next property and return it."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyPropertiesIteratorType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.PropertiesIterator", - .tp_basicsize = sizeof(PyPropertiesIterator), - .tp_dealloc = reinterpret_cast(PyPropertiesIteratorDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_properties_iterator.", - .tp_methods = PyPropertiesIteratorMethods, -}; -// clang-format on - -// clang-format off -struct PyEdge { - PyObject_HEAD - mgp_edge *edge; - PyGraph *py_graph; -}; -// clang-format on - -PyObject *PyEdgeGetTypeName(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edge_type edge_type{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_edge_get_type(self->edge, &edge_type))) { - return nullptr; - } - return PyUnicode_FromString(edge_type.name); -} - -PyObject *PyEdgeFromVertex(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - return MakePyVertex(self->edge->from, self->py_graph); -} - -PyObject *PyEdgeToVertex(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - return MakePyVertex(self->edge->to, self->py_graph); -} - -void PyEdgeDealloc(PyEdge *self) { - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_edge_destroy` if we are not in valid execution context. - // The query execution should free all memory used during execution, so we may - // cause a double free issue. - if (self->py_graph->graph) mgp_edge_destroy(self->edge); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyEdgeIsValid(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(self->py_graph && self->py_graph->graph); -} - -PyObject *PyEdgeUnderlyingGraphIsMutable(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - return PyBool_FromLong(CallBool(mgp_graph_is_mutable, self->py_graph->graph)); -} - -PyObject *PyEdgeGetId(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edge_id edge_id{0}; - if (RaiseExceptionFromErrorCode(mgp_edge_get_id(self->edge, &edge_id))) { - return nullptr; - } - return PyLong_FromLongLong(edge_id.as_int); -} - -PyObject *PyEdgeIterProperties(PyEdge *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_properties_iterator *properties_it{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_edge_iter_properties(self->edge, self->py_graph->memory, &properties_it))) { - return nullptr; - } - auto *py_properties_it = PyObject_New(PyPropertiesIterator, &PyPropertiesIteratorType); - if (!py_properties_it) { - mgp_properties_iterator_destroy(properties_it); - return nullptr; - } - py_properties_it->it = properties_it; - Py_INCREF(self->py_graph); - py_properties_it->py_graph = self->py_graph; - return reinterpret_cast(py_properties_it); -} - -PyObject *PyEdgeGetProperty(PyEdge *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *prop_name = nullptr; - if (!PyArg_ParseTuple(args, "s", &prop_name)) return nullptr; - mgp_value *prop_value{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_edge_get_property(self->edge, prop_name, self->py_graph->memory, &prop_value))) { - return nullptr; - } - auto py_prop_value = MgpValueToPyObject(*prop_value, self->py_graph); - mgp_value_destroy(prop_value); - return py_prop_value.Steal(); -} - -PyObject *PyEdgeSetProperty(PyEdge *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->edge); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *prop_name = nullptr; - PyObject *py_value{nullptr}; - if (!PyArg_ParseTuple(args, "sO", &prop_name, &py_value)) { - return nullptr; - } - MgpUniquePtr prop_value{PyObjectToMgpValueWithPythonExceptions(py_value, self->py_graph->memory), - mgp_value_destroy}; - - if (prop_value == nullptr || - RaiseExceptionFromErrorCode(mgp_edge_set_property(self->edge, prop_name, prop_value.get()))) { - return nullptr; - } - Py_RETURN_NONE; -} - -static PyMethodDef PyEdgeMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported."}, - {"is_valid", reinterpret_cast(PyEdgeIsValid), METH_NOARGS, - "Return True if the edge is in valid context and may be used."}, - {"underlying_graph_is_mutable", reinterpret_cast(PyEdgeUnderlyingGraphIsMutable), METH_NOARGS, - "Return True if the edge is mutable and can be modified."}, - {"get_id", reinterpret_cast(PyEdgeGetId), METH_NOARGS, "Return edge id."}, - {"get_type_name", reinterpret_cast(PyEdgeGetTypeName), METH_NOARGS, "Return the edge's type name."}, - {"from_vertex", reinterpret_cast(PyEdgeFromVertex), METH_NOARGS, "Return the edge's source vertex."}, - {"to_vertex", reinterpret_cast(PyEdgeToVertex), METH_NOARGS, "Return the edge's destination vertex."}, - {"iter_properties", reinterpret_cast(PyEdgeIterProperties), METH_NOARGS, - "Return _mgp.PropertiesIterator for this edge."}, - {"get_property", reinterpret_cast(PyEdgeGetProperty), METH_VARARGS, - "Return edge property with given name."}, - {"set_property", reinterpret_cast(PyEdgeSetProperty), METH_VARARGS, - "Set the value of the property on the edge."}, - {nullptr}, -}; - -PyObject *PyEdgeRichCompare(PyObject *self, PyObject *other, int op); - -// clang-format off -static PyTypeObject PyEdgeType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Edge", - .tp_basicsize = sizeof(PyEdge), - .tp_dealloc = reinterpret_cast(PyEdgeDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_edge.", - .tp_richcompare = PyEdgeRichCompare, - .tp_methods = PyEdgeMethods, -}; -// clang-format on - -PyObject *MakePyEdgeWithoutCopy(mgp_edge &edge, PyGraph *py_graph) { - MG_ASSERT(py_graph); - MG_ASSERT(py_graph->graph && py_graph->memory); - MG_ASSERT(edge.GetMemoryResource() == py_graph->memory->impl); - auto *py_edge = PyObject_New(PyEdge, &PyEdgeType); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - if (!py_edge) return nullptr; - py_edge->edge = &edge; - py_edge->py_graph = py_graph; - Py_INCREF(py_graph); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - return reinterpret_cast(py_edge); -} - -/// Create an instance of `_mgp.Edge` class. -/// -/// The created instance references an existing `_mgp.Graph` instance, which -/// marks the execution context. -PyObject *MakePyEdge(mgp_edge &edge, PyGraph *py_graph) { - MG_ASSERT(py_graph); - MG_ASSERT(py_graph->graph && py_graph->memory); - MgpUniquePtr edge_copy{nullptr, mgp_edge_destroy}; - if (RaiseExceptionFromErrorCode(CreateMgpObject(edge_copy, mgp_edge_copy, &edge, py_graph->memory))) { - return nullptr; - } - auto *py_edge = MakePyEdgeWithoutCopy(*edge_copy, py_graph); - if (py_edge != nullptr) { - static_cast(edge_copy.release()); - } - return py_edge; -} - -PyObject *PyEdgeRichCompare(PyObject *self, PyObject *other, int op) { - MG_ASSERT(self); - MG_ASSERT(other); - - if (Py_TYPE(self) != &PyEdgeType || Py_TYPE(other) != &PyEdgeType || op != Py_EQ) { - Py_RETURN_NOTIMPLEMENTED; - } - - auto *e1 = reinterpret_cast(self); - auto *e2 = reinterpret_cast(other); - MG_ASSERT(e1->edge); - MG_ASSERT(e2->edge); - return PyBool_FromLong(Call(mgp_edge_equal, e1->edge, e2->edge)); -} - -// clang-format off -struct PyVertex { - PyObject_HEAD - mgp_vertex *vertex; - PyGraph *py_graph; -}; -// clang-format on - -void PyVertexDealloc(PyVertex *self) { - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_vertex_destroy` if we are not in valid execution - // context. The query execution should free all memory used during - // execution, so we may cause a double free issue. - if (self->py_graph->graph) mgp_vertex_destroy(self->vertex); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyVertexIsValid(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(self->py_graph && self->py_graph->graph); -} - -PyObject *PyVertexUnderlyingGraphIsMutable(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - return PyBool_FromLong(CallBool(mgp_graph_is_mutable, self->py_graph->graph)); -} - -PyObject *PyVertexGetId(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_vertex_id id{}; - if (RaiseExceptionFromErrorCode(mgp_vertex_get_id(self->vertex, &id))) { - return nullptr; - } - return PyLong_FromLongLong(id.as_int); -} - -PyObject *PyVertexLabelsCount(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - size_t label_count{0}; - if (RaiseExceptionFromErrorCode(mgp_vertex_labels_count(self->vertex, &label_count))) { - return nullptr; - } - return PyLong_FromSize_t(label_count); -} - -PyObject *PyVertexLabelAt(PyVertex *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - static_assert(std::numeric_limits::max() <= std::numeric_limits::max()); - Py_ssize_t id; - if (!PyArg_ParseTuple(args, "n", &id)) { - return nullptr; - } - mgp_label label{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertex_label_at(self->vertex, id, &label))) { - return nullptr; - } - return PyUnicode_FromString(label.name); -} - -PyObject *PyVertexIterInEdges(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edges_iterator *edges_it{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertex_iter_in_edges(self->vertex, self->py_graph->memory, &edges_it))) { - return nullptr; - } - auto *py_edges_it = PyObject_New(PyEdgesIterator, &PyEdgesIteratorType); - if (!py_edges_it) { - mgp_edges_iterator_destroy(edges_it); - return nullptr; - } - py_edges_it->it = edges_it; - Py_INCREF(self->py_graph); - py_edges_it->py_graph = self->py_graph; - return reinterpret_cast(py_edges_it); -} - -PyObject *PyVertexIterOutEdges(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_edges_iterator *edges_it{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertex_iter_out_edges(self->vertex, self->py_graph->memory, &edges_it))) { - return nullptr; - } - auto *py_edges_it = PyObject_New(PyEdgesIterator, &PyEdgesIteratorType); - if (!py_edges_it) { - mgp_edges_iterator_destroy(edges_it); - return nullptr; - } - py_edges_it->it = edges_it; - Py_INCREF(self->py_graph); - py_edges_it->py_graph = self->py_graph; - return reinterpret_cast(py_edges_it); -} - -PyObject *PyVertexIterProperties(PyVertex *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - mgp_properties_iterator *properties_it{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_vertex_iter_properties(self->vertex, self->py_graph->memory, &properties_it))) { - return nullptr; - } - auto *py_properties_it = PyObject_New(PyPropertiesIterator, &PyPropertiesIteratorType); - if (!py_properties_it) { - mgp_properties_iterator_destroy(properties_it); - return nullptr; - } - py_properties_it->it = properties_it; - Py_INCREF(self->py_graph); - py_properties_it->py_graph = self->py_graph; - return reinterpret_cast(py_properties_it); -} - -PyObject *PyVertexGetProperty(PyVertex *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *prop_name{nullptr}; - if (!PyArg_ParseTuple(args, "s", &prop_name)) { - return nullptr; - } - mgp_value *prop_value{nullptr}; - if (RaiseExceptionFromErrorCode( - mgp_vertex_get_property(self->vertex, prop_name, self->py_graph->memory, &prop_value))) { - return nullptr; - } - auto py_prop_value = MgpValueToPyObject(*prop_value, self->py_graph); - mgp_value_destroy(prop_value); - return py_prop_value.Steal(); -} - -PyObject *PyVertexSetProperty(PyVertex *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *prop_name = nullptr; - PyObject *py_value{nullptr}; - if (!PyArg_ParseTuple(args, "sO", &prop_name, &py_value)) { - return nullptr; - } - MgpUniquePtr prop_value{PyObjectToMgpValueWithPythonExceptions(py_value, self->py_graph->memory), - mgp_value_destroy}; - - if (prop_value == nullptr || - RaiseExceptionFromErrorCode(mgp_vertex_set_property(self->vertex, prop_name, prop_value.get()))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyVertexAddLabel(PyVertex *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *label_name = nullptr; - if (!PyArg_ParseTuple(args, "s", &label_name)) { - return nullptr; - } - if (RaiseExceptionFromErrorCode(mgp_vertex_add_label(self->vertex, mgp_label{label_name}))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyVertexRemoveLabel(PyVertex *self, PyObject *args) { - MG_ASSERT(self); - MG_ASSERT(self->vertex); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - const char *label_name = nullptr; - if (!PyArg_ParseTuple(args, "s", &label_name)) { - return nullptr; - } - if (RaiseExceptionFromErrorCode(mgp_vertex_remove_label(self->vertex, mgp_label{label_name}))) { - return nullptr; - } - Py_RETURN_NONE; -} - -static PyMethodDef PyVertexMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported."}, - {"is_valid", reinterpret_cast(PyVertexIsValid), METH_NOARGS, - "Return True if the vertex is in valid context and may be used."}, - {"underlying_graph_is_mutable", reinterpret_cast(PyVertexUnderlyingGraphIsMutable), METH_NOARGS, - "Return True if the vertex is mutable and can be modified."}, - {"get_id", reinterpret_cast(PyVertexGetId), METH_NOARGS, "Return vertex id."}, - {"labels_count", reinterpret_cast(PyVertexLabelsCount), METH_NOARGS, - "Return number of lables of a vertex."}, - {"label_at", reinterpret_cast(PyVertexLabelAt), METH_VARARGS, - "Return label of a vertex on a given index."}, - {"add_label", reinterpret_cast(PyVertexAddLabel), METH_VARARGS, "Add the label to the vertex."}, - {"remove_label", reinterpret_cast(PyVertexRemoveLabel), METH_VARARGS, - "Remove the label from the vertex."}, - {"iter_in_edges", reinterpret_cast(PyVertexIterInEdges), METH_NOARGS, - "Return _mgp.EdgesIterator for in edges."}, - {"iter_out_edges", reinterpret_cast(PyVertexIterOutEdges), METH_NOARGS, - "Return _mgp.EdgesIterator for out edges."}, - {"iter_properties", reinterpret_cast(PyVertexIterProperties), METH_NOARGS, - "Return _mgp.PropertiesIterator for this vertex."}, - {"get_property", reinterpret_cast(PyVertexGetProperty), METH_VARARGS, - "Return vertex property with given name."}, - {"set_property", reinterpret_cast(PyVertexSetProperty), METH_VARARGS, - "Set the value of the property on the vertex."}, - {nullptr}, -}; - -PyObject *PyVertexRichCompare(PyObject *self, PyObject *other, int op); - -// clang-format off -static PyTypeObject PyVertexType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Vertex", - .tp_basicsize = sizeof(PyVertex), - .tp_dealloc = reinterpret_cast(PyVertexDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_vertex.", - .tp_richcompare = PyVertexRichCompare, - .tp_methods = PyVertexMethods, -}; -// clang-format on - -PyObject *MakePyVertexWithoutCopy(mgp_vertex &vertex, PyGraph *py_graph) { - MG_ASSERT(py_graph); - MG_ASSERT(py_graph->graph && py_graph->memory); - MG_ASSERT(vertex.GetMemoryResource() == py_graph->memory->impl); - auto *py_vertex = PyObject_New(PyVertex, &PyVertexType); - if (!py_vertex) return nullptr; - py_vertex->vertex = &vertex; - py_vertex->py_graph = py_graph; - Py_INCREF(py_graph); - return reinterpret_cast(py_vertex); -} - -PyObject *MakePyVertex(mgp_vertex &vertex, PyGraph *py_graph) { - MG_ASSERT(py_graph); - MG_ASSERT(py_graph->graph && py_graph->memory); - - MgpUniquePtr vertex_copy{nullptr, mgp_vertex_destroy}; - if (RaiseExceptionFromErrorCode(CreateMgpObject(vertex_copy, mgp_vertex_copy, &vertex, py_graph->memory))) { - return nullptr; - } - auto *py_vertex = MakePyVertexWithoutCopy(*vertex_copy, py_graph); - if (py_vertex != nullptr) { - static_cast(vertex_copy.release()); - } - return py_vertex; -} - -PyObject *PyVertexRichCompare(PyObject *self, PyObject *other, int op) { - MG_ASSERT(self); - MG_ASSERT(other); - - if (Py_TYPE(self) != &PyVertexType || Py_TYPE(other) != &PyVertexType || op != Py_EQ) { - Py_RETURN_NOTIMPLEMENTED; - } - - auto *v1 = reinterpret_cast(self); - auto *v2 = reinterpret_cast(other); - MG_ASSERT(v1->vertex); - MG_ASSERT(v2->vertex); - - return PyBool_FromLong(Call(mgp_vertex_equal, v1->vertex, v2->vertex)); -} - -// clang-format off -struct PyPath { - PyObject_HEAD - mgp_path *path; - PyGraph *py_graph; -}; -// clang-format on - -void PyPathDealloc(PyPath *self) { - MG_ASSERT(self->path); - MG_ASSERT(self->py_graph); - // Avoid invoking `mgp_path_destroy` if we are not in valid execution - // context. The query execution should free all memory used during - // execution, so we may cause a double free issue. - if (self->py_graph->graph) mgp_path_destroy(self->path); - Py_DECREF(self->py_graph); - Py_TYPE(self)->tp_free(self); -} - -PyObject *PyPathIsValid(PyPath *self, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong(self->py_graph && self->py_graph->graph); -} - -PyObject *PyPathMakeWithStart(PyTypeObject *type, PyObject *vertex); - -PyObject *PyPathExpand(PyPath *self, PyObject *edge) { - MG_ASSERT(self->path); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - if (Py_TYPE(edge) != &PyEdgeType) { - PyErr_SetString(PyExc_TypeError, "Expected a _mgp.Edge."); - return nullptr; - } - auto *py_edge = reinterpret_cast(edge); - - if (RaiseExceptionFromErrorCode(mgp_path_expand(self->path, py_edge->edge))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyPathSize(PyPath *self, PyObject *Py_UNUSED(ignored)) { - MG_ASSERT(self->path); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - return PyLong_FromSize_t(Call(mgp_path_size, self->path)); -} - -PyObject *PyPathVertexAt(PyPath *self, PyObject *args) { - MG_ASSERT(self->path); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - static_assert(std::numeric_limits::max() <= std::numeric_limits::max()); - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "n", &i)) { - return nullptr; - } - mgp_vertex *vertex{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_path_vertex_at(self->path, i, &vertex))) { - return nullptr; - } - return MakePyVertex(*vertex, self->py_graph); -} - -PyObject *PyPathEdgeAt(PyPath *self, PyObject *args) { - MG_ASSERT(self->path); - MG_ASSERT(self->py_graph); - MG_ASSERT(self->py_graph->graph); - static_assert(std::numeric_limits::max() <= std::numeric_limits::max()); - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "n", &i)) { - return nullptr; - } - mgp_edge *edge{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_path_edge_at(self->path, i, &edge))) { - return nullptr; - } - return MakePyEdge(*edge, self->py_graph); -} - -static PyMethodDef PyPathMethods[] = { - {"__reduce__", reinterpret_cast(DisallowPickleAndCopy), METH_NOARGS, "__reduce__ is not supported"}, - {"is_valid", reinterpret_cast(PyPathIsValid), METH_NOARGS, - "Return True if Path is in valid context and may be used."}, - {"make_with_start", reinterpret_cast(PyPathMakeWithStart), METH_O | METH_CLASS, - "Create a path with a starting vertex."}, - {"expand", reinterpret_cast(PyPathExpand), METH_O, - "Append an edge continuing from the last vertex on the path."}, - {"size", reinterpret_cast(PyPathSize), METH_NOARGS, "Return the number of edges in a mgp_path."}, - {"vertex_at", reinterpret_cast(PyPathVertexAt), METH_VARARGS, - "Return the vertex from a path at given index."}, - {"edge_at", reinterpret_cast(PyPathEdgeAt), METH_VARARGS, - "Return the edge from a path at given index."}, - {nullptr}, -}; - -// clang-format off -static PyTypeObject PyPathType = { - PyVarObject_HEAD_INIT(nullptr, 0) - .tp_name = "_mgp.Path", - .tp_basicsize = sizeof(PyPath), - .tp_dealloc = reinterpret_cast(PyPathDealloc), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Wraps struct mgp_path.", - .tp_methods = PyPathMethods, -}; -// clang-format on - -PyObject *MakePyPath(mgp_path *path, PyGraph *py_graph) { - MG_ASSERT(path); - MG_ASSERT(py_graph->graph && py_graph->memory); - MG_ASSERT(path->GetMemoryResource() == py_graph->memory->impl); - auto *py_path = PyObject_New(PyPath, &PyPathType); - if (!py_path) return nullptr; - py_path->path = path; - py_path->py_graph = py_graph; - Py_INCREF(py_graph); - return reinterpret_cast(py_path); -} - -PyObject *MakePyPath(mgp_path &path, PyGraph *py_graph) { - MG_ASSERT(py_graph); - MG_ASSERT(py_graph->graph && py_graph->memory); - mgp_path *path_copy{nullptr}; - - if (RaiseExceptionFromErrorCode(mgp_path_copy(&path, py_graph->memory, &path_copy))) { - return nullptr; - } - auto *py_path = MakePyPath(path_copy, py_graph); - if (!py_path) mgp_path_destroy(path_copy); - return py_path; -} - -PyObject *PyPathMakeWithStart(PyTypeObject *type, PyObject *vertex) { - if (type != &PyPathType) { - PyErr_SetString(PyExc_TypeError, "Expected '' as the first argument."); - return nullptr; - } - if (Py_TYPE(vertex) != &PyVertexType) { - PyErr_SetString(PyExc_TypeError, "Expected a '_mgp.Vertex' as the second argument."); - return nullptr; - } - auto *py_vertex = reinterpret_cast(vertex); - mgp_path *path{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_path_make_with_start(py_vertex->vertex, py_vertex->py_graph->memory, &path))) { - return nullptr; - } - auto *py_path = MakePyPath(path, py_vertex->py_graph); - if (!py_path) mgp_path_destroy(path); - return py_path; -} - -struct PyMgpError { - const char *name; - PyObject *&exception; - PyObject *&base; - const char *docstring; -}; - -bool AddModuleConstants(PyObject &module) { - // add source type constants - if (PyModule_AddIntConstant(&module, "SOURCE_TYPE_KAFKA", static_cast(mgp_source_type::KAFKA))) { - return false; - } - if (PyModule_AddIntConstant(&module, "SOURCE_TYPE_PULSAR", static_cast(mgp_source_type::PULSAR))) { - return false; - } - - return true; -} - -PyObject *PyInitMgpModule() { - PyObject *mgp = PyModule_Create(&PyMgpModule); - if (!mgp) return nullptr; - auto register_type = [mgp](auto *type, const auto *name) -> bool { - if (PyType_Ready(type) < 0) { - Py_DECREF(mgp); - return false; - } - Py_INCREF(type); - if (PyModule_AddObject(mgp, name, reinterpret_cast(type)) < 0) { - Py_DECREF(type); - Py_DECREF(mgp); - return false; - } - return true; - }; - - if (!AddModuleConstants(*mgp)) return nullptr; - - if (!register_type(&PyPropertiesIteratorType, "PropertiesIterator")) return nullptr; - if (!register_type(&PyVerticesIteratorType, "VerticesIterator")) return nullptr; - if (!register_type(&PyEdgesIteratorType, "EdgesIterator")) return nullptr; - if (!register_type(&PyGraphType, "Graph")) return nullptr; - if (!register_type(&PyEdgeType, "Edge")) return nullptr; - if (!register_type(&PyQueryProcType, "Proc")) return nullptr; - if (!register_type(&PyMagicFuncType, "Func")) return nullptr; - if (!register_type(&PyQueryModuleType, "Module")) return nullptr; - if (!register_type(&PyVertexType, "Vertex")) return nullptr; - if (!register_type(&PyPathType, "Path")) return nullptr; - if (!register_type(&PyCypherTypeType, "Type")) return nullptr; - if (!register_type(&PyMessagesType, "Messages")) return nullptr; - if (!register_type(&PyMessageType, "Message")) return nullptr; - - std::array py_mgp_errors{ - PyMgpError{"_mgp.UnknownError", gMgpUnknownError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.UnableToAllocateError", gMgpUnableToAllocateError, PyExc_MemoryError, nullptr}, - PyMgpError{"_mgp.InsufficientBufferError", gMgpInsufficientBufferError, PyExc_BufferError, nullptr}, - PyMgpError{"_mgp.OutOfRangeError", gMgpOutOfRangeError, PyExc_BufferError, nullptr}, - PyMgpError{"_mgp.LogicErrorError", gMgpLogicErrorError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.DeletedObjectError", gMgpDeletedObjectError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.InvalidArgumentError", gMgpInvalidArgumentError, PyExc_ValueError, nullptr}, - PyMgpError{"_mgp.KeyAlreadyExistsError", gMgpKeyAlreadyExistsError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.ImmutableObjectError", gMgpImmutableObjectError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.ValueConversionError", gMgpValueConversionError, PyExc_RuntimeError, nullptr}, - PyMgpError{"_mgp.SerializationError", gMgpSerializationError, PyExc_RuntimeError, nullptr}, - }; - Py_INCREF(Py_None); - - utils::OnScopeExit clean_up{[mgp, &py_mgp_errors] { - for (const auto &py_mgp_error : py_mgp_errors) { - Py_XDECREF(py_mgp_error.exception); - } - Py_DECREF(Py_None); - Py_DECREF(mgp); - }}; - - if (PyModule_AddObject(mgp, "_MODULE", Py_None) < 0) { - return nullptr; - } - - auto register_custom_error = [mgp](PyMgpError &py_mgp_error) { - py_mgp_error.exception = PyErr_NewException(py_mgp_error.name, py_mgp_error.base, nullptr); - if (py_mgp_error.exception == nullptr) { - return false; - } - - const auto *name_in_module = std::string_view(py_mgp_error.name).substr(5).data(); - return PyModule_AddObject(mgp, name_in_module, py_mgp_error.exception) == 0; - }; - - for (auto &py_mgp_error : py_mgp_errors) { - if (!register_custom_error(py_mgp_error)) { - return nullptr; - } - } - clean_up.Disable(); - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) - PyDateTime_IMPORT; - - return mgp; -} - -namespace { - -template -auto WithMgpModule(mgp_module *module_def, const TFun &fun) { - py::Object py_mgp(PyImport_ImportModule("_mgp")); - MG_ASSERT(py_mgp, "Expected builtin '_mgp' to be available for import"); - py::Object py_mgp_module(py_mgp.GetAttr("_MODULE")); - MG_ASSERT(py_mgp_module, "Expected '_mgp' to have attribute '_MODULE'"); - // NOTE: This check is not thread safe, but this should only go through - // ModuleRegistry::LoadModuleLibrary which ought to serialize loading. - MG_ASSERT(py_mgp_module.Ptr() == Py_None, - "Expected '_mgp._MODULE' to be None as we are just starting to " - "import a new module. Is some other thread also importing Python " - "modules?"); - auto *py_query_module = MakePyQueryModule(module_def); - MG_ASSERT(py_query_module); - MG_ASSERT(py_mgp.SetAttr("_MODULE", py_query_module)); - auto ret = fun(); - auto maybe_exc = py::FetchError(); - MG_ASSERT(py_mgp.SetAttr("_MODULE", Py_None)); - if (maybe_exc) { - py::RestoreError(*maybe_exc); - } - return ret; -} - -} // namespace - -py::Object ImportPyModule(const char *name, mgp_module *module_def) { - return WithMgpModule(module_def, [name]() { return py::Object(PyImport_ImportModule(name)); }); -} - -py::Object ReloadPyModule(PyObject *py_module, mgp_module *module_def) { - return WithMgpModule(module_def, [py_module]() { return py::Object(PyImport_ReloadModule(py_module)); }); -} - -py::Object MgpValueToPyObject(const mgp_value &value, PyObject *py_graph) { - if (Py_TYPE(py_graph) != &PyGraphType) { - PyErr_SetString(PyExc_TypeError, "Expected a _mgp.Graph."); - return nullptr; - } - return MgpValueToPyObject(value, reinterpret_cast(py_graph)); -} - -py::Object MgpValueToPyObject(const mgp_value &value, PyGraph *py_graph) { - switch (value.type) { - case MGP_VALUE_TYPE_NULL: - Py_INCREF(Py_None); - return py::Object(Py_None); - case MGP_VALUE_TYPE_BOOL: - return py::Object(PyBool_FromLong(value.bool_v)); - case MGP_VALUE_TYPE_INT: - return py::Object(PyLong_FromLongLong(value.int_v)); - case MGP_VALUE_TYPE_DOUBLE: - return py::Object(PyFloat_FromDouble(value.double_v)); - case MGP_VALUE_TYPE_STRING: - return py::Object(PyUnicode_FromString(value.string_v.c_str())); - case MGP_VALUE_TYPE_LIST: - return MgpListToPyTuple(value.list_v, py_graph); - case MGP_VALUE_TYPE_MAP: { - auto *map = value.map_v; - py::Object py_dict(PyDict_New()); - if (!py_dict) { - return nullptr; - } - for (const auto &[key, val] : map->items) { - auto py_val = MgpValueToPyObject(val, py_graph); - if (!py_val) { - return nullptr; - } - // Unlike PyList_SET_ITEM, PyDict_SetItem does not steal the value. - if (PyDict_SetItemString(py_dict.Ptr(), key.c_str(), py_val.Ptr()) != 0) return nullptr; - } - return py_dict; - } - case MGP_VALUE_TYPE_VERTEX: { - py::Object py_mgp(PyImport_ImportModule("mgp")); - if (!py_mgp) return nullptr; - auto *v = value.vertex_v; - py::Object py_vertex(reinterpret_cast(MakePyVertex(*v, py_graph))); - return py_mgp.CallMethod("Vertex", py_vertex); - } - case MGP_VALUE_TYPE_EDGE: { - py::Object py_mgp(PyImport_ImportModule("mgp")); - if (!py_mgp) return nullptr; - auto *e = value.edge_v; - py::Object py_edge(reinterpret_cast(MakePyEdge(*e, py_graph))); - return py_mgp.CallMethod("Edge", py_edge); - } - case MGP_VALUE_TYPE_PATH: { - py::Object py_mgp(PyImport_ImportModule("mgp")); - if (!py_mgp) return nullptr; - auto *p = value.path_v; - py::Object py_path(reinterpret_cast(MakePyPath(*p, py_graph))); - return py_mgp.CallMethod("Path", py_path); - } - case MGP_VALUE_TYPE_DATE: { - const auto &date = value.date_v->date; - py::Object py_date(PyDate_FromDate(date.year, date.month, date.day)); - return py_date; - } - case MGP_VALUE_TYPE_LOCAL_TIME: { - const auto &local_time = value.local_time_v->local_time; - py::Object py_local_time(PyTime_FromTime(local_time.hour, local_time.minute, local_time.second, - local_time.millisecond * 1000 + local_time.microsecond)); - return py_local_time; - } - case MGP_VALUE_TYPE_LOCAL_DATE_TIME: { - const auto &local_time = value.local_date_time_v->local_date_time.local_time; - const auto &date = value.local_date_time_v->local_date_time.date; - py::Object py_local_date_time(PyDateTime_FromDateAndTime(date.year, date.month, date.day, local_time.hour, - local_time.minute, local_time.second, - local_time.millisecond * 1000 + local_time.microsecond)); - return py_local_date_time; - } - case MGP_VALUE_TYPE_DURATION: { - const auto &duration = value.duration_v->duration; - py::Object py_duration(PyDelta_FromDSU(0, 0, duration.microseconds)); - return py_duration; - } - } -} - -mgp_value *PyObjectToMgpValue(PyObject *o, mgp_memory *memory) { - auto py_seq_to_list = [memory](PyObject *seq, Py_ssize_t len, const auto &py_seq_get_item) { - static_assert(std::numeric_limits::max() <= std::numeric_limits::max()); - MgpUniquePtr list{nullptr, &mgp_list_destroy}; - if (const auto err = CreateMgpObject(list, mgp_list_make_empty, len, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during making mgp_list"}; - } - for (Py_ssize_t i = 0; i < len; ++i) { - PyObject *e = py_seq_get_item(seq, i); - mgp_value *v{nullptr}; - v = PyObjectToMgpValue(e, memory); - const auto err = mgp_list_append(list.get(), v); - mgp_value_destroy(v); - if (err != mgp_error::MGP_ERROR_NO_ERROR) { - if (err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } - throw std::runtime_error{"Unexpected error during appending to mgp_list"}; - } - } - mgp_value *v{nullptr}; - if (const auto err = mgp_value_make_list(list.get(), &v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during making mgp_value"}; - } - static_cast(list.release()); - return v; - }; - - auto is_mgp_instance = [](PyObject *obj, const char *mgp_type_name) { - py::Object py_mgp(PyImport_ImportModule("mgp")); - if (!py_mgp) { - PyErr_Clear(); - // This way we skip conversions of types from user-facing 'mgp' module. - return false; - } - auto mgp_type = py_mgp.GetAttr(mgp_type_name); - if (!mgp_type) { - PyErr_Clear(); - std::stringstream ss; - ss << "'mgp' module is missing '" << mgp_type_name << "' type"; - throw std::invalid_argument(ss.str()); - } - int res = PyObject_IsInstance(obj, mgp_type.Ptr()); - if (res == -1) { - PyErr_Clear(); - std::stringstream ss; - ss << "Error when checking object is instance of 'mgp." << mgp_type_name << "' type"; - throw std::invalid_argument(ss.str()); - } - return static_cast(res); - }; - - mgp_value *mgp_v{nullptr}; - mgp_error last_error{mgp_error::MGP_ERROR_NO_ERROR}; - - if (o == Py_None) { - last_error = mgp_value_make_null(memory, &mgp_v); - } else if (PyBool_Check(o)) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) Py_True is defined with C-style cast - last_error = mgp_value_make_bool(static_cast(o == Py_True), memory, &mgp_v); - } else if (PyLong_Check(o)) { - int64_t value = PyLong_AsLong(o); - if (PyErr_Occurred()) { - PyErr_Clear(); - throw std::overflow_error("Python integer is out of range"); - } - last_error = mgp_value_make_int(value, memory, &mgp_v); - } else if (PyFloat_Check(o)) { - last_error = mgp_value_make_double(PyFloat_AsDouble(o), memory, &mgp_v); - } else if (PyUnicode_Check(o)) { // NOLINT(hicpp-signed-bitwise) - last_error = mgp_value_make_string(PyUnicode_AsUTF8(o), memory, &mgp_v); - } else if (PyList_Check(o)) { - mgp_v = py_seq_to_list(o, PyList_Size(o), [](auto *list, const auto i) { return PyList_GET_ITEM(list, i); }); - } else if (PyTuple_Check(o)) { - mgp_v = py_seq_to_list(o, PyTuple_Size(o), [](auto *tuple, const auto i) { return PyTuple_GET_ITEM(tuple, i); }); - } else if (PyDict_Check(o)) { // NOLINT(hicpp-signed-bitwise) - MgpUniquePtr map{nullptr, mgp_map_destroy}; - const auto map_err = CreateMgpObject(map, mgp_map_make_empty, memory); - - if (map_err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } - if (map_err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during creating mgp_map"}; - } - - PyObject *key{nullptr}; - PyObject *value{nullptr}; - Py_ssize_t pos{0}; - while (PyDict_Next(o, &pos, &key, &value)) { - if (!PyUnicode_Check(key)) { - throw std::invalid_argument("Dictionary keys must be strings"); - } - - const char *k = PyUnicode_AsUTF8(key); - - if (!k) { - PyErr_Clear(); - throw std::bad_alloc{}; - } - - MgpUniquePtr v{PyObjectToMgpValue(value, memory), mgp_value_destroy}; - - if (const auto err = mgp_map_insert(map.get(), k, v.get()); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during inserting an item to mgp_map"}; - } - } - - if (const auto err = mgp_value_make_map(map.get(), &mgp_v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during creating mgp_value"}; - } - static_cast(map.release()); - } else if (Py_TYPE(o) == &PyEdgeType) { - MgpUniquePtr e{nullptr, mgp_edge_destroy}; - // Copy the edge and pass the ownership to the created mgp_value. - - if (const auto err = CreateMgpObject(e, mgp_edge_copy, reinterpret_cast(o)->edge, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_edge"}; - } - if (const auto err = mgp_value_make_edge(e.get(), &mgp_v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_edge"}; - } - static_cast(e.release()); - } else if (Py_TYPE(o) == &PyPathType) { - MgpUniquePtr p{nullptr, mgp_path_destroy}; - // Copy the edge and pass the ownership to the created mgp_value. - - if (const auto err = CreateMgpObject(p, mgp_path_copy, reinterpret_cast(o)->path, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_path"}; - } - if (const auto err = mgp_value_make_path(p.get(), &mgp_v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_path"}; - } - static_cast(p.release()); - } else if (Py_TYPE(o) == &PyVertexType) { - MgpUniquePtr v{nullptr, mgp_vertex_destroy}; - // Copy the edge and pass the ownership to the created mgp_value. - - if (const auto err = CreateMgpObject(v, mgp_vertex_copy, reinterpret_cast(o)->vertex, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_vertex"}; - } - if (const auto err = mgp_value_make_vertex(v.get(), &mgp_v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error during copying mgp_vertex"}; - } - static_cast(v.release()); - } else if (is_mgp_instance(o, "Edge")) { - py::Object edge(PyObject_GetAttrString(o, "_edge")); - if (!edge) { - PyErr_Clear(); - throw std::invalid_argument("'mgp.Edge' is missing '_edge' attribute"); - } - return PyObjectToMgpValue(edge.Ptr(), memory); - } else if (is_mgp_instance(o, "Vertex")) { - py::Object vertex(PyObject_GetAttrString(o, "_vertex")); - if (!vertex) { - PyErr_Clear(); - throw std::invalid_argument("'mgp.Vertex' is missing '_vertex' attribute"); - } - return PyObjectToMgpValue(vertex.Ptr(), memory); - } else if (is_mgp_instance(o, "Path")) { - py::Object path(PyObject_GetAttrString(o, "_path")); - if (!path) { - PyErr_Clear(); - throw std::invalid_argument("'mgp.Path' is missing '_path' attribute"); - } - return PyObjectToMgpValue(path.Ptr(), memory); - } else if (PyDate_CheckExact(o)) { - mgp_date_parameters parameters{ - .year = PyDateTime_GET_YEAR(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .month = PyDateTime_GET_MONTH(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .day = PyDateTime_GET_DAY(o)}; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - MgpUniquePtr date{nullptr, mgp_date_destroy}; - - if (const auto err = CreateMgpObject(date, mgp_date_from_parameters, ¶meters, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_date"}; - } - if (const auto err = mgp_value_make_date(date.get(), &mgp_v); err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_value"}; - } - static_cast(date.release()); - } else if (PyTime_CheckExact(o)) { - mgp_local_time_parameters parameters{ - .hour = PyDateTime_TIME_GET_HOUR(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .minute = PyDateTime_TIME_GET_MINUTE(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .second = PyDateTime_TIME_GET_SECOND(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .millisecond = - PyDateTime_TIME_GET_MICROSECOND(o) / // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - 1000, - .microsecond = - PyDateTime_TIME_GET_MICROSECOND(o) % // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - 1000}; - MgpUniquePtr local_time{nullptr, mgp_local_time_destroy}; - - if (const auto err = CreateMgpObject(local_time, mgp_local_time_from_parameters, ¶meters, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_local_time"}; - } - if (const auto err = mgp_value_make_local_time(local_time.get(), &mgp_v); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_value"}; - } - static_cast(local_time.release()); - } else if (PyDateTime_CheckExact(o)) { - mgp_date_parameters date_parameters{ - .year = PyDateTime_GET_YEAR(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .month = PyDateTime_GET_MONTH(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .day = PyDateTime_GET_DAY(o)}; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - mgp_local_time_parameters local_time_parameters{ - .hour = PyDateTime_DATE_GET_HOUR(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .minute = PyDateTime_DATE_GET_MINUTE(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .second = PyDateTime_DATE_GET_SECOND(o), // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - .millisecond = - PyDateTime_DATE_GET_MICROSECOND(o) / // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - 1000, - .microsecond = - PyDateTime_DATE_GET_MICROSECOND(o) % // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - 1000}; - - mgp_local_date_time_parameters parameters{&date_parameters, &local_time_parameters}; - - MgpUniquePtr local_date_time{nullptr, mgp_local_date_time_destroy}; - - if (const auto err = CreateMgpObject(local_date_time, mgp_local_date_time_from_parameters, ¶meters, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_local_date_time"}; - } - if (const auto err = mgp_value_make_local_date_time(local_date_time.get(), &mgp_v); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_value"}; - } - static_cast(local_date_time.release()); - } else if (PyDelta_CheckExact(o)) { - static constexpr int64_t microseconds_in_days = - static_cast(std::chrono::days{1}).count(); - const auto days = - PyDateTime_DELTA_GET_DAYS(o); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - auto microseconds = - std::abs(days) * microseconds_in_days + - PyDateTime_DELTA_GET_SECONDS(o) * 1000 * // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - 1000 + - PyDateTime_DELTA_GET_MICROSECONDS(o); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,hicpp-signed-bitwise) - microseconds *= days < 0 ? -1 : 1; - - MgpUniquePtr duration{nullptr, mgp_duration_destroy}; - - if (const auto err = CreateMgpObject(duration, mgp_duration_from_microseconds, microseconds, memory); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_duration"}; - } - if (const auto err = mgp_value_make_duration(duration.get(), &mgp_v); - err == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } else if (err != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_value"}; - } - static_cast(duration.release()); - } else { - throw std::invalid_argument("Unsupported PyObject conversion"); - } - - if (last_error == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { - throw std::bad_alloc{}; - } - if (last_error != mgp_error::MGP_ERROR_NO_ERROR) { - throw std::runtime_error{"Unexpected error while creating mgp_value"}; - } - - return mgp_v; -} - -PyObject *PyGraphCreateEdge(PyGraph *self, PyObject *args) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - PyVertex *from{nullptr}; - PyVertex *to{nullptr}; - const char *edge_type{nullptr}; - if (!PyArg_ParseTuple(args, "O!O!s", &PyVertexType, &from, &PyVertexType, &to, &edge_type)) { - return nullptr; - } - MgpUniquePtr new_edge{nullptr, mgp_edge_destroy}; - if (RaiseExceptionFromErrorCode(CreateMgpObject(new_edge, mgp_graph_create_edge, self->graph, from->vertex, - to->vertex, mgp_edge_type{edge_type}, self->memory))) { - return nullptr; - } - auto *py_edge = MakePyEdgeWithoutCopy(*new_edge, self); - if (py_edge != nullptr) { - static_cast(new_edge.release()); - } - return py_edge; -} - -PyObject *PyGraphDeleteVertex(PyGraph *self, PyObject *args) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - PyVertex *vertex{nullptr}; - if (!PyArg_ParseTuple(args, "O!", &PyVertexType, &vertex)) { - return nullptr; - } - if (RaiseExceptionFromErrorCode(mgp_graph_delete_vertex(self->graph, vertex->vertex))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyGraphDetachDeleteVertex(PyGraph *self, PyObject *args) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - PyVertex *vertex{nullptr}; - if (!PyArg_ParseTuple(args, "O!", &PyVertexType, &vertex)) { - return nullptr; - } - if (RaiseExceptionFromErrorCode(mgp_graph_detach_delete_vertex(self->graph, vertex->vertex))) { - return nullptr; - } - Py_RETURN_NONE; -} - -PyObject *PyGraphDeleteEdge(PyGraph *self, PyObject *args) { - MG_ASSERT(PyGraphIsValidImpl(*self)); - MG_ASSERT(self->memory); - PyEdge *edge{nullptr}; - if (!PyArg_ParseTuple(args, "O!", &PyEdgeType, &edge)) { - return nullptr; - } - if (RaiseExceptionFromErrorCode(mgp_graph_delete_edge(self->graph, edge->edge))) { - return nullptr; - } - Py_RETURN_NONE; -} - -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/procedure/py_module.hpp b/src/query/v2/procedure/py_module.hpp deleted file mode 100644 index b037f7797..000000000 --- a/src/query/v2/procedure/py_module.hpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/// @file -/// Functions and types for loading Query Modules written in Python. -#pragma once - -#include "py/py.hpp" - -struct mgp_graph; -struct mgp_memory; -struct mgp_module; -struct mgp_value; - -namespace memgraph::query::v2::procedure { - -struct PyGraph; - -/// Convert an `mgp_value` into a Python object, referencing the given `PyGraph` -/// instance and using the same allocator as the graph. -/// -/// Values of type `MGP_VALUE_TYPE_VERTEX`, `MGP_VALUE_TYPE_EDGE` and -/// `MGP_VALUE_TYPE_PATH` are returned as `mgp.Vertex`, `mgp.Edge` and -/// `mgp.Path` respectively, and *not* their internal `_mgp` -/// representations. Other value types are converted to equivalent builtin -/// Python objects. -/// -/// Return a non-null `py::Object` instance on success. Otherwise, return a null -/// `py::Object` instance and set the appropriate Python exception. -py::Object MgpValueToPyObject(const mgp_value &value, PyGraph *py_graph); - -py::Object MgpValueToPyObject(const mgp_value &value, PyObject *py_graph); - -/// Convert a Python object into `mgp_value`, constructing it using the given -/// `mgp_memory` allocator. -/// -/// If the user-facing 'mgp' module can be imported, this function will handle -/// conversion of 'mgp.Vertex', 'mgp.Edge' and 'mgp.Path' values. -/// -/// @throw std::bad_alloc -/// @throw std::overflow_error if attempting to convert a Python integer which -/// too large to fit into int64_t. -/// @throw std::invalid_argument if the given Python object cannot be converted -/// to an mgp_value (e.g. a dictionary whose keys aren't strings or an object -/// of unsupported type). -mgp_value *PyObjectToMgpValue(PyObject *, mgp_memory *); - -/// Create the _mgp module for use in embedded Python. -/// -/// The function is to be used before Py_Initialize via the following code. -/// -/// PyImport_AppendInittab("_mgp", &query::v2::procedure::PyInitMgpModule); -PyObject *PyInitMgpModule(); - -/// Create an instance of _mgp.Graph class. -PyObject *MakePyGraph(mgp_graph *, mgp_memory *); - -/// Import a module with given name in the context of mgp_module. -/// -/// This function can only be called when '_mgp' module has been initialized in -/// Python. -/// -/// Return nullptr and set appropriate Python exception on failure. -py::Object ImportPyModule(const char *, mgp_module *); - -/// Reload already loaded Python module in the context of mgp_module. -/// -/// This function can only be called when '_mgp' module has been initialized in -/// Python. -/// -/// Return nullptr and set appropriate Python exception on failure. -py::Object ReloadPyModule(PyObject *, mgp_module *); - -} // namespace memgraph::query::v2::procedure diff --git a/src/query/v2/requests.hpp b/src/query/v2/requests.hpp index b16c44541..80e688c9f 100644 --- a/src/query/v2/requests.hpp +++ b/src/query/v2/requests.hpp @@ -47,6 +47,7 @@ inline bool operator==(const VertexId &lhs, const VertexId &rhs) { using Gid = size_t; using PropertyId = memgraph::storage::v3::PropertyId; +using EdgeTypeId = memgraph::storage::v3::EdgeTypeId; struct EdgeType { uint64_t id; diff --git a/src/query/v2/shard_request_manager.hpp b/src/query/v2/shard_request_manager.hpp index e5d279ecc..a5de5a9ae 100644 --- a/src/query/v2/shard_request_manager.hpp +++ b/src/query/v2/shard_request_manager.hpp @@ -105,12 +105,19 @@ class ShardRequestManagerInterface { virtual ~ShardRequestManagerInterface() = default; virtual void StartTransaction() = 0; + virtual void Commit() = 0; virtual std::vector Request(ExecutionState &state) = 0; virtual std::vector Request(ExecutionState &state, std::vector new_vertices) = 0; virtual std::vector Request(ExecutionState &state) = 0; - virtual memgraph::storage::v3::PropertyId NameToProperty(const std::string &name) const = 0; - virtual memgraph::storage::v3::LabelId LabelNameToLabelId(const std::string &name) const = 0; + // TODO(antaljanosbenjamin): unify the GetXXXId and NameToId functions to have consistent naming, return type and + // implementation + virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0; + virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0; + virtual storage::v3::LabelId LabelNameToLabelId(const std::string &name) const = 0; + virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0; + virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0; + virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0; virtual bool IsPrimaryKey(PropertyId name) const = 0; }; @@ -155,7 +162,48 @@ class ShardRequestManager : public ShardRequestManagerInterface { } } - memgraph::storage::v3::PropertyId NameToProperty(const std::string &name) const override { + void Commit() override { + memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()}; + CoordinatorWriteRequests write_req = req; + auto write_res = coord_cli_.SendWriteRequest(write_req); + if (write_res.HasError()) { + throw std::runtime_error("HLC request for commit failed"); + } + auto coordinator_write_response = write_res.GetValue(); + auto hlc_response = std::get(coordinator_write_response); + + if (hlc_response.fresher_shard_map) { + shards_map_ = hlc_response.fresher_shard_map.value(); + } + auto commit_timestamp = hlc_response.new_hlc; + + msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp}; + + for (const auto &[label, space] : shards_map_.label_spaces) { + for (const auto &[key, shard] : space.shards) { + auto &storage_client = GetStorageClientForShard(shard, label); + // TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture + // instead. + auto commit_response = storage_client.SendWriteRequest(commit_req); + // RETRY on timeouts? + // Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test + if (commit_response.HasError()) { + throw std::runtime_error("Commit request timed out"); + } + WriteResponses write_response_variant = commit_response.GetValue(); + auto &response = std::get(write_response_variant); + if (!response.success) { + throw std::runtime_error("Commit request did not succeed"); + } + } + } + } + + storage::v3::EdgeTypeId NameToEdgeType(const std::string & /*name*/) const override { + return memgraph::storage::v3::EdgeTypeId::FromUint(0); + } + + storage::v3::PropertyId NameToProperty(const std::string &name) const override { return *shards_map_.GetPropertyId(name); } @@ -163,6 +211,19 @@ class ShardRequestManager : public ShardRequestManagerInterface { return shards_map_.GetLabelId(name); } + const std::string &PropertyToName(memgraph::storage::v3::PropertyId /*prop*/) const override { + static std::string str{"dummy__prop"}; + return str; + } + const std::string &LabelToName(memgraph::storage::v3::LabelId /*label*/) const override { + static std::string str{"dummy__label"}; + return str; + } + const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId /*type*/) const override { + static std::string str{"dummy__edgetype"}; + return str; + } + bool IsPrimaryKey(const PropertyId name) const override { return std::find_if(shards_map_.properties.begin(), shards_map_.properties.end(), [name](auto &pr) { return pr.second == name; }) != shards_map_.properties.end(); @@ -222,7 +283,6 @@ class ShardRequestManager : public ShardRequestManagerInterface { auto primary_key = state.requests[id].new_vertices[0].primary_key; auto &storage_client = GetStorageClientForShard(*shard_it, labels[0].id); WriteRequests req = state.requests[id]; - auto ladaksd = std::get(req); auto write_response_result = storage_client.SendWriteRequest(req); // RETRY on timeouts? // Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test diff --git a/src/query/v2/stream/common.cpp b/src/query/v2/stream/common.cpp deleted file mode 100644 index 268899b9a..000000000 --- a/src/query/v2/stream/common.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/stream/common.hpp" - -#include - -namespace memgraph::query::v2::stream { -namespace { -const std::string kBatchIntervalKey{"batch_interval"}; -const std::string kBatchSizeKey{"batch_size"}; -const std::string kTransformationName{"transformation_name"}; -} // namespace - -void to_json(nlohmann::json &data, CommonStreamInfo &&common_info) { - data[kBatchIntervalKey] = common_info.batch_interval.count(); - data[kBatchSizeKey] = common_info.batch_size; - data[kTransformationName] = common_info.transformation_name; -} - -void from_json(const nlohmann::json &data, CommonStreamInfo &common_info) { - if (const auto batch_interval = data.at(kBatchIntervalKey); !batch_interval.is_null()) { - using BatchInterval = decltype(common_info.batch_interval); - common_info.batch_interval = BatchInterval{batch_interval.get()}; - } else { - common_info.batch_interval = kDefaultBatchInterval; - } - - if (const auto batch_size = data.at(kBatchSizeKey); !batch_size.is_null()) { - common_info.batch_size = batch_size.get(); - } else { - common_info.batch_size = kDefaultBatchSize; - } - - data.at(kTransformationName).get_to(common_info.transformation_name); -} -} // namespace memgraph::query::v2::stream diff --git a/src/query/v2/stream/common.hpp b/src/query/v2/stream/common.hpp deleted file mode 100644 index ee8761545..000000000 --- a/src/query/v2/stream/common.hpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "query/v2/procedure/mg_procedure_impl.hpp" - -namespace memgraph::query::v2::stream { - -inline constexpr std::chrono::milliseconds kDefaultBatchInterval{100}; -inline constexpr int64_t kDefaultBatchSize{1000}; - -template -using ConsumerFunction = std::function &)>; - -struct CommonStreamInfo { - std::chrono::milliseconds batch_interval; - int64_t batch_size; - std::string transformation_name; -}; - -template -concept ConvertableToJson = requires(T value, nlohmann::json data) { - { to_json(data, std::move(value)) } -> std::same_as; - { from_json(data, value) } -> std::same_as; -}; - -template -concept ConvertableToMgpMessage = requires(T value) { - mgp_message{value}; -}; - -template -concept Stream = requires(TStream stream) { - typename TStream::StreamInfo; - typename TStream::Message; - TStream{std::string{""}, typename TStream::StreamInfo{}, ConsumerFunction{}}; - { stream.Start() } -> std::same_as; - { stream.StartWithLimit(uint64_t{}, std::optional{}) } -> std::same_as; - { stream.Stop() } -> std::same_as; - { stream.IsRunning() } -> std::same_as; - { - stream.Check(std::optional{}, std::optional{}, - ConsumerFunction{}) - } -> std::same_as; - requires std::same_as().common_info)>, - CommonStreamInfo>; - - requires ConvertableToMgpMessage; - requires ConvertableToJson; -}; - -enum class StreamSourceType : uint8_t { KAFKA, PULSAR }; - -constexpr std::string_view StreamSourceTypeToString(StreamSourceType type) { - switch (type) { - case StreamSourceType::KAFKA: - return "kafka"; - case StreamSourceType::PULSAR: - return "pulsar"; - } -} - -template -StreamSourceType StreamType(const T & /*stream*/); - -const std::string kCommonInfoKey = "common_info"; - -void to_json(nlohmann::json &data, CommonStreamInfo &&info); -void from_json(const nlohmann::json &data, CommonStreamInfo &common_info); -} // namespace memgraph::query::v2::stream diff --git a/src/query/v2/stream/sources.cpp b/src/query/v2/stream/sources.cpp deleted file mode 100644 index 686bc4d56..000000000 --- a/src/query/v2/stream/sources.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/stream/sources.hpp" - -#include - -#include "integrations/constants.hpp" - -namespace memgraph::query::v2::stream { -KafkaStream::KafkaStream(std::string stream_name, StreamInfo stream_info, - ConsumerFunction consumer_function) { - integrations::kafka::ConsumerInfo consumer_info{ - .consumer_name = std::move(stream_name), - .topics = std::move(stream_info.topics), - .consumer_group = std::move(stream_info.consumer_group), - .bootstrap_servers = std::move(stream_info.bootstrap_servers), - .batch_interval = stream_info.common_info.batch_interval, - .batch_size = stream_info.common_info.batch_size, - .public_configs = std::move(stream_info.configs), - .private_configs = std::move(stream_info.credentials), - }; - consumer_.emplace(std::move(consumer_info), std::move(consumer_function)); -}; - -KafkaStream::StreamInfo KafkaStream::Info(std::string transformation_name) const { - const auto &info = consumer_->Info(); - return {{.batch_interval = info.batch_interval, - .batch_size = info.batch_size, - .transformation_name = std::move(transformation_name)}, - .topics = info.topics, - .consumer_group = info.consumer_group, - .bootstrap_servers = info.bootstrap_servers, - .configs = info.public_configs, - .credentials = info.private_configs}; -} - -void KafkaStream::Start() { consumer_->Start(); } -void KafkaStream::StartWithLimit(uint64_t batch_limit, std::optional timeout) const { - consumer_->StartWithLimit(batch_limit, timeout); -} -void KafkaStream::Stop() { consumer_->Stop(); } -bool KafkaStream::IsRunning() const { return consumer_->IsRunning(); } - -void KafkaStream::Check(std::optional timeout, std::optional batch_limit, - const ConsumerFunction &consumer_function) const { - consumer_->Check(timeout, batch_limit, consumer_function); -} - -utils::BasicResult KafkaStream::SetStreamOffset(const int64_t offset) { - return consumer_->SetConsumerOffsets(offset); -} - -namespace { -const std::string kTopicsKey{"topics"}; -const std::string kConsumerGroupKey{"consumer_group"}; -const std::string kBoostrapServers{"bootstrap_servers"}; -const std::string kConfigs{"configs"}; -const std::string kCredentials{"credentials"}; - -const std::unordered_map kDefaultConfigsMap; -} // namespace - -void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info) { - data[kCommonInfoKey] = std::move(info.common_info); - data[kTopicsKey] = std::move(info.topics); - data[kConsumerGroupKey] = info.consumer_group; - data[kBoostrapServers] = std::move(info.bootstrap_servers); - data[kConfigs] = std::move(info.configs); - data[kCredentials] = std::move(info.credentials); -} - -void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info) { - data.at(kCommonInfoKey).get_to(info.common_info); - data.at(kTopicsKey).get_to(info.topics); - data.at(kConsumerGroupKey).get_to(info.consumer_group); - data.at(kBoostrapServers).get_to(info.bootstrap_servers); - // These values might not be present in the persisted JSON object - info.configs = data.value(kConfigs, kDefaultConfigsMap); - info.credentials = data.value(kCredentials, kDefaultConfigsMap); -} - -PulsarStream::PulsarStream(std::string stream_name, StreamInfo stream_info, - ConsumerFunction consumer_function) { - integrations::pulsar::ConsumerInfo consumer_info{.batch_size = stream_info.common_info.batch_size, - .batch_interval = stream_info.common_info.batch_interval, - .topics = std::move(stream_info.topics), - .consumer_name = std::move(stream_name), - .service_url = std::move(stream_info.service_url)}; - - consumer_.emplace(std::move(consumer_info), std::move(consumer_function)); -}; - -PulsarStream::StreamInfo PulsarStream::Info(std::string transformation_name) const { - const auto &info = consumer_->Info(); - return {{.batch_interval = info.batch_interval, - .batch_size = info.batch_size, - .transformation_name = std::move(transformation_name)}, - .topics = info.topics, - .service_url = info.service_url}; -} - -void PulsarStream::Start() { consumer_->Start(); } -void PulsarStream::StartWithLimit(uint64_t batch_limit, std::optional timeout) const { - consumer_->StartWithLimit(batch_limit, timeout); -} -void PulsarStream::Stop() { consumer_->Stop(); } -bool PulsarStream::IsRunning() const { return consumer_->IsRunning(); } -void PulsarStream::Check(std::optional timeout, std::optional batch_limit, - const ConsumerFunction &consumer_function) const { - consumer_->Check(timeout, batch_limit, consumer_function); -} - -namespace { -const std::string kServiceUrl{"service_url"}; -} // namespace - -void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info) { - data[kCommonInfoKey] = std::move(info.common_info); - data[kTopicsKey] = std::move(info.topics); - data[kServiceUrl] = std::move(info.service_url); -} - -void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info) { - data.at(kCommonInfoKey).get_to(info.common_info); - data.at(kTopicsKey).get_to(info.topics); - data.at(kServiceUrl).get_to(info.service_url); -} -} // namespace memgraph::query::v2::stream diff --git a/src/query/v2/stream/sources.hpp b/src/query/v2/stream/sources.hpp deleted file mode 100644 index ba5b75dc0..000000000 --- a/src/query/v2/stream/sources.hpp +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include "query/v2/stream/common.hpp" - -#include "integrations/kafka/consumer.hpp" -#include "integrations/pulsar/consumer.hpp" - -namespace memgraph::query::v2::stream { - -struct KafkaStream { - struct StreamInfo { - CommonStreamInfo common_info; - std::vector topics; - std::string consumer_group; - std::string bootstrap_servers; - std::unordered_map configs; - std::unordered_map credentials; - }; - - using Message = integrations::kafka::Message; - - KafkaStream(std::string stream_name, StreamInfo stream_info, - ConsumerFunction consumer_function); - - StreamInfo Info(std::string transformation_name) const; - - void Start(); - void StartWithLimit(uint64_t batch_limit, std::optional timeout) const; - void Stop(); - bool IsRunning() const; - - void Check(std::optional timeout, std::optional batch_limit, - const ConsumerFunction &consumer_function) const; - - utils::BasicResult SetStreamOffset(int64_t offset); - - private: - using Consumer = integrations::kafka::Consumer; - std::optional consumer_; -}; - -void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info); -void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info); - -template <> -inline StreamSourceType StreamType(const KafkaStream & /*stream*/) { - return StreamSourceType::KAFKA; -} - -struct PulsarStream { - struct StreamInfo { - CommonStreamInfo common_info; - std::vector topics; - std::string service_url; - }; - - using Message = integrations::pulsar::Message; - - PulsarStream(std::string stream_name, StreamInfo stream_info, ConsumerFunction consumer_function); - - StreamInfo Info(std::string transformation_name) const; - - void Start(); - void StartWithLimit(uint64_t batch_limit, std::optional timeout) const; - void Stop(); - bool IsRunning() const; - - void Check(std::optional timeout, std::optional batch_limit, - const ConsumerFunction &consumer_function) const; - - private: - using Consumer = integrations::pulsar::Consumer; - std::optional consumer_; -}; - -void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info); -void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info); - -template <> -inline StreamSourceType StreamType(const PulsarStream & /*stream*/) { - return StreamSourceType::PULSAR; -} - -} // namespace memgraph::query::v2::stream diff --git a/src/query/v2/stream/streams.cpp b/src/query/v2/stream/streams.cpp deleted file mode 100644 index b7e6abe61..000000000 --- a/src/query/v2/stream/streams.cpp +++ /dev/null @@ -1,773 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "query/v2/stream/streams.hpp" - -#include -#include -#include - -#include -#include - -#include "integrations/constants.hpp" -#include "mg_procedure.h" -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/db_accessor.hpp" -#include "query/v2/discard_value_stream.hpp" -#include "query/v2/exceptions.hpp" -#include "query/v2/interpreter.hpp" -#include "query/v2/procedure/mg_procedure_helpers.hpp" -#include "query/v2/procedure/mg_procedure_impl.hpp" -#include "query/v2/procedure/module.hpp" -#include "query/v2/stream/sources.hpp" -#include "storage/v3/conversions.hpp" -#include "utils/event_counter.hpp" -#include "utils/logging.hpp" -#include "utils/memory.hpp" -#include "utils/on_scope_exit.hpp" -#include "utils/pmr/string.hpp" -#include "utils/variant_helpers.hpp" - -namespace EventCounter { -extern const Event MessagesConsumed; -} // namespace EventCounter - -namespace memgraph::query::v2::stream { -namespace { -inline constexpr auto kExpectedTransformationResultSize = 2; -inline constexpr auto kCheckStreamResultSize = 2; -const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()}; -const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()}; - -const std::map empty_parameters{}; - -auto GetStream(auto &map, const std::string &stream_name) { - if (auto it = map.find(stream_name); it != map.end()) { - return it; - } - throw StreamsException("Couldn't find stream '{}'", stream_name); -} - -std::pair ExtractTransformationResult( - const utils::pmr::map &values, const std::string_view transformation_name, - const std::string_view stream_name) { - if (values.size() != kExpectedTransformationResultSize) { - throw StreamsException( - "Transformation '{}' in stream '{}' did not yield all fields (query, parameters) as required.", - transformation_name, stream_name); - } - - auto get_value = [&](const utils::pmr::string &field_name) mutable -> const TypedValue & { - auto it = values.find(field_name); - if (it == values.end()) { - throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.", - transformation_name, stream_name, field_name}; - }; - return it->second; - }; - - const auto &query_value = get_value(query_param_name); - MG_ASSERT(query_value.IsString()); - const auto ¶ms_value = get_value(params_param_name); - MG_ASSERT(params_value.IsNull() || params_value.IsMap()); - return {query_value, params_value}; -} - -template -void CallCustomTransformation(const std::string &transformation_name, const std::vector &messages, - mgp_result &result, storage::v3::Shard::Accessor &storage_accessor, - utils::MemoryResource &memory_resource, const std::string &stream_name) { - DbAccessor db_accessor{&storage_accessor}; - { - auto maybe_transformation = - procedure::FindTransformation(procedure::gModuleRegistry, transformation_name, utils::NewDeleteResource()); - - if (!maybe_transformation) { - throw StreamsException("Couldn't find transformation {} for stream '{}'", transformation_name, stream_name); - }; - const auto &trans = *maybe_transformation->second; - mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}}; - std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages), - [](const TMessage &message) { return mgp_message{message}; }); - mgp_graph graph{&db_accessor, storage::v3::View::OLD, nullptr}; - mgp_memory memory{&memory_resource}; - result.rows.clear(); - result.error_msg.reset(); - result.signature = &trans.results; - - MG_ASSERT(result.signature->size() == kExpectedTransformationResultSize); - MG_ASSERT(result.signature->contains(query_param_name)); - MG_ASSERT(result.signature->contains(params_param_name)); - - spdlog::trace("Calling transformation in stream '{}'", stream_name); - trans.cb(&mgp_messages, &graph, &result, &memory); - } - if (result.error_msg.has_value()) { - throw StreamsException(result.error_msg->c_str()); - } -} - -template -StreamStatus CreateStatus(std::string stream_name, std::string transformation_name, - std::optional owner, const TStream &stream) { - return {.name = std::move(stream_name), - .type = StreamType(stream), - .is_running = stream.IsRunning(), - .info = stream.Info(std::move(transformation_name)), - .owner = std::move(owner)}; -} - -// nlohmann::json doesn't support string_view access yet -const std::string kStreamName{"name"}; -const std::string kIsRunningKey{"is_running"}; -const std::string kOwner{"owner"}; -const std::string kType{"type"}; -} // namespace - -template -void to_json(nlohmann::json &data, StreamStatus &&status) { - data[kStreamName] = std::move(status.name); - data[kType] = status.type; - data[kIsRunningKey] = status.is_running; - - if (status.owner.has_value()) { - data[kOwner] = std::move(*status.owner); - } else { - data[kOwner] = nullptr; - } - - to_json(data, std::move(status.info)); -} - -template -void from_json(const nlohmann::json &data, StreamStatus &status) { - data.at(kStreamName).get_to(status.name); - data.at(kIsRunningKey).get_to(status.is_running); - - if (const auto &owner = data.at(kOwner); !owner.is_null()) { - status.owner = owner.get(); - } else { - status.owner = {}; - } - - from_json(data, status.info); -} - -Streams::Streams(InterpreterContext *interpreter_context, std::filesystem::path directory) - : interpreter_context_(interpreter_context), storage_(std::move(directory)) { - RegisterProcedures(); -} - -void Streams::RegisterProcedures() { - RegisterKafkaProcedures(); - RegisterPulsarProcedures(); -} - -void Streams::RegisterKafkaProcedures() { - { - static constexpr std::string_view proc_name = "kafka_set_stream_offset"; - auto set_stream_offset = [this](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result, - mgp_memory * /*memory*/) { - auto *arg_stream_name = procedure::Call(mgp_list_at, args, 0); - const auto *stream_name = procedure::Call(mgp_value_get_string, arg_stream_name); - auto *arg_offset = procedure::Call(mgp_list_at, args, 1); - const auto offset = procedure::Call(mgp_value_get_int, arg_offset); - auto lock_ptr = streams_.Lock(); - auto it = GetStream(*lock_ptr, std::string(stream_name)); - std::visit(utils::Overloaded{[&](StreamData &kafka_stream) { - auto stream_source_ptr = kafka_stream.stream_source->Lock(); - const auto error = stream_source_ptr->SetStreamOffset(offset); - if (error.HasError()) { - MG_ASSERT(mgp_result_set_error_msg(result, error.GetError().c_str()) == - mgp_error::MGP_ERROR_NO_ERROR, - "Unable to set procedure error message of procedure: {}", proc_name); - } - }, - [](auto && /*other*/) { - throw QueryRuntimeException("'{}' can be only used for Kafka stream sources", - proc_name); - }}, - it->second); - }; - - mgp_proc proc(proc_name, set_stream_offset, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_arg(&proc, "offset", procedure::Call(mgp_type_int)) == - mgp_error::MGP_ERROR_NO_ERROR); - - procedure::gModuleRegistry.RegisterMgProcedure(proc_name, std::move(proc)); - } - - { - static constexpr std::string_view proc_name = "kafka_stream_info"; - - static constexpr std::string_view consumer_group_result_name = "consumer_group"; - static constexpr std::string_view topics_result_name = "topics"; - static constexpr std::string_view bootstrap_servers_result_name = "bootstrap_servers"; - static constexpr std::string_view configs_result_name = "configs"; - static constexpr std::string_view credentials_result_name = "credentials"; - - auto get_stream_info = [this](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result, mgp_memory *memory) { - auto *arg_stream_name = procedure::Call(mgp_list_at, args, 0); - const auto *stream_name = procedure::Call(mgp_value_get_string, arg_stream_name); - auto lock_ptr = streams_.Lock(); - auto it = GetStream(*lock_ptr, std::string(stream_name)); - std::visit( - utils::Overloaded{ - [&](StreamData &kafka_stream) { - auto stream_source_ptr = kafka_stream.stream_source->Lock(); - const auto info = stream_source_ptr->Info(kafka_stream.transformation_name); - mgp_result_record *record{nullptr}; - if (!procedure::TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - const auto consumer_group_value = - procedure::GetStringValueOrSetError(info.consumer_group.c_str(), memory, result); - if (!consumer_group_value) { - return; - } - - procedure::MgpUniquePtr topic_names{nullptr, mgp_list_destroy}; - if (!procedure::TryOrSetError( - [&] { - return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(), - memory); - }, - result)) { - return; - } - - for (const auto &topic : info.topics) { - auto topic_value = procedure::GetStringValueOrSetError(topic.c_str(), memory, result); - if (!topic_value) { - return; - } - topic_names->elems.push_back(std::move(*topic_value)); - } - - procedure::MgpUniquePtr topics_value{nullptr, mgp_value_destroy}; - if (!procedure::TryOrSetError( - [&] { - return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.get()); - }, - result)) { - return; - } - static_cast(topic_names.release()); - - const auto bootstrap_servers_value = - procedure::GetStringValueOrSetError(info.bootstrap_servers.c_str(), memory, result); - if (!bootstrap_servers_value) { - return; - } - - const auto convert_config_map = - [result, memory](const std::unordered_map &configs_to_convert) - -> procedure::MgpUniquePtr { - procedure::MgpUniquePtr configs_value{nullptr, mgp_value_destroy}; - procedure::MgpUniquePtr configs{nullptr, mgp_map_destroy}; - if (!procedure::TryOrSetError( - [&] { return procedure::CreateMgpObject(configs, mgp_map_make_empty, memory); }, result)) { - return configs_value; - } - - for (const auto &[key, value] : configs_to_convert) { - auto value_value = procedure::GetStringValueOrSetError(value.c_str(), memory, result); - if (!value_value) { - return configs_value; - } - configs->items.emplace(key, std::move(*value_value)); - } - - if (!procedure::TryOrSetError( - [&] { return procedure::CreateMgpObject(configs_value, mgp_value_make_map, configs.get()); }, - result)) { - return configs_value; - } - static_cast(configs.release()); - return configs_value; - }; - - const auto configs_value = convert_config_map(info.configs); - if (configs_value == nullptr) { - return; - } - - using CredentialsType = decltype(KafkaStream::StreamInfo::credentials); - CredentialsType reducted_credentials; - std::transform(info.credentials.begin(), info.credentials.end(), - std::inserter(reducted_credentials, reducted_credentials.end()), - [](const auto &pair) -> CredentialsType::value_type { - return {pair.first, integrations::kReducted}; - }); - - const auto credentials_value = convert_config_map(reducted_credentials); - if (credentials_value == nullptr) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, consumer_group_result_name.data(), - consumer_group_value.get())) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, bootstrap_servers_result_name.data(), - bootstrap_servers_value.get())) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, configs_result_name.data(), - configs_value.get())) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, credentials_result_name.data(), - credentials_value.get())) { - return; - } - }, - [](auto && /*other*/) { - throw QueryRuntimeException("'{}' can be only used for Kafka stream sources", proc_name); - }}, - it->second); - }; - - mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&proc, consumer_group_result_name.data(), - procedure::Call(mgp_type_string)) == mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT( - mgp_proc_add_result(&proc, topics_result_name.data(), - procedure::Call(mgp_type_list, procedure::Call(mgp_type_string))) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&proc, bootstrap_servers_result_name.data(), - procedure::Call(mgp_type_string)) == mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&proc, configs_result_name.data(), procedure::Call(mgp_type_map)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&proc, credentials_result_name.data(), procedure::Call(mgp_type_map)) == - mgp_error::MGP_ERROR_NO_ERROR); - - procedure::gModuleRegistry.RegisterMgProcedure(proc_name, std::move(proc)); - } -} - -void Streams::RegisterPulsarProcedures() { - { - static constexpr std::string_view proc_name = "pulsar_stream_info"; - static constexpr std::string_view service_url_result_name = "service_url"; - static constexpr std::string_view topics_result_name = "topics"; - auto get_stream_info = [this](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result, mgp_memory *memory) { - auto *arg_stream_name = procedure::Call(mgp_list_at, args, 0); - const auto *stream_name = procedure::Call(mgp_value_get_string, arg_stream_name); - auto lock_ptr = streams_.Lock(); - auto it = GetStream(*lock_ptr, std::string(stream_name)); - std::visit( - utils::Overloaded{ - [&](StreamData &pulsar_stream) { - auto stream_source_ptr = pulsar_stream.stream_source->Lock(); - const auto info = stream_source_ptr->Info(pulsar_stream.transformation_name); - mgp_result_record *record{nullptr}; - if (!procedure::TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) { - return; - } - - auto service_url_value = procedure::GetStringValueOrSetError(info.service_url.c_str(), memory, result); - if (!service_url_value) { - return; - } - - procedure::MgpUniquePtr topic_names{nullptr, mgp_list_destroy}; - if (!procedure::TryOrSetError( - [&] { - return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(), - memory); - }, - result)) { - return; - } - - for (const auto &topic : info.topics) { - auto topic_value = procedure::GetStringValueOrSetError(topic.c_str(), memory, result); - if (!topic_value) { - return; - } - topic_names->elems.push_back(std::move(*topic_value)); - } - - procedure::MgpUniquePtr topics_value{nullptr, mgp_value_destroy}; - if (!procedure::TryOrSetError( - [&] { - return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.release()); - }, - result)) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) { - return; - } - - if (!procedure::InsertResultOrSetError(result, record, service_url_result_name.data(), - service_url_value.get())) { - return; - } - }, - [](auto && /*other*/) { - throw QueryRuntimeException("'{}' can be only used for Pulsar stream sources", proc_name); - }}, - it->second); - }; - - mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource()); - MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call(mgp_type_string)) == - mgp_error::MGP_ERROR_NO_ERROR); - MG_ASSERT(mgp_proc_add_result(&proc, service_url_result_name.data(), - procedure::Call(mgp_type_string)) == mgp_error::MGP_ERROR_NO_ERROR); - - MG_ASSERT( - mgp_proc_add_result(&proc, topics_result_name.data(), - procedure::Call(mgp_type_list, procedure::Call(mgp_type_string))) == - mgp_error::MGP_ERROR_NO_ERROR); - - procedure::gModuleRegistry.RegisterMgProcedure(proc_name, std::move(proc)); - } -} - -template -void Streams::Create(const std::string &stream_name, typename TStream::StreamInfo info, - std::optional owner) { - auto locked_streams = streams_.Lock(); - auto it = CreateConsumer(*locked_streams, stream_name, std::move(info), std::move(owner)); - - try { - std::visit( - [&](const auto &stream_data) { - const auto stream_source_ptr = stream_data.stream_source->ReadLock(); - Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr)); - }, - it->second); - } catch (...) { - locked_streams->erase(it); - throw; - } -} - -template void Streams::Create(const std::string &stream_name, KafkaStream::StreamInfo info, - std::optional owner); -template void Streams::Create(const std::string &stream_name, PulsarStream::StreamInfo info, - std::optional owner); - -template -Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std::string &stream_name, - typename TStream::StreamInfo stream_info, - std::optional owner) { - if (map.contains(stream_name)) { - throw StreamsException{"Stream already exists with name '{}'", stream_name}; - } - - auto *memory_resource = utils::NewDeleteResource(); - - auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, stream_name, - transformation_name = stream_info.common_info.transformation_name, owner = owner, - interpreter = std::make_shared(interpreter_context_), - result = mgp_result{nullptr, memory_resource}, - total_retries = interpreter_context_->config.stream_transaction_conflict_retries, - retry_interval = interpreter_context_->config.stream_transaction_retry_interval]( - const std::vector &messages) mutable { - auto accessor = interpreter_context->db->Access(coordinator::Hlc{}); - EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size()); - CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name); - - DiscardValueResultStream stream; - - spdlog::trace("Start transaction in stream '{}'", stream_name); - utils::OnScopeExit cleanup{[&interpreter, &result]() { - result.rows.clear(); - interpreter->Abort(); - }}; - - const static std::map empty_parameters{}; - uint32_t i = 0; - while (true) { - try { - interpreter->BeginTransaction(); - for (auto &row : result.rows) { - spdlog::trace("Processing row in stream '{}'", stream_name); - auto [query_value, params_value] = ExtractTransformationResult(row.values, transformation_name, stream_name); - storage::v3::PropertyValue params_prop = storage::v3::TypedToPropertyValue(params_value); - - std::string query{query_value.ValueString()}; - spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name); - auto prepare_result = - interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr); - if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges)) { - throw StreamsException{ - "Couldn't execute query '{}' for stream '{}' because the owner is not authorized to execute the " - "query!", - query, stream_name}; - } - interpreter->PullAll(&stream); - } - - spdlog::trace("Commit transaction in stream '{}'", stream_name); - interpreter->CommitTransaction(); - result.rows.clear(); - break; - } catch (const query::v2::TransactionSerializationException &e) { - interpreter->Abort(); - if (i == total_retries) { - throw; - } - ++i; - std::this_thread::sleep_for(retry_interval); - } - } - }; - - auto insert_result = map.try_emplace( - stream_name, StreamData{std::move(stream_info.common_info.transformation_name), std::move(owner), - std::make_unique>( - stream_name, std::move(stream_info), std::move(consumer_function))}); - MG_ASSERT(insert_result.second, "Unexpected error during storing consumer '{}'", stream_name); - return insert_result.first; -} - -void Streams::RestoreStreams() { - spdlog::info("Loading streams..."); - auto locked_streams_map = streams_.Lock(); - MG_ASSERT(locked_streams_map->empty(), "Cannot restore streams when some streams already exist!"); - - for (const auto &[stream_name, stream_data] : storage_) { - const auto get_failed_message = [&stream_name = stream_name](const std::string_view message, - const std::string_view nested_message) { - return fmt::format("Failed to load stream '{}', because: {} caused by {}", stream_name, message, nested_message); - }; - - const auto create_consumer = [&, &stream_name = stream_name, this](StreamStatus status, - auto &&stream_json_data) { - try { - stream_json_data.get_to(status); - } catch (const nlohmann::json::type_error &exception) { - spdlog::warn(get_failed_message("invalid type conversion", exception.what())); - return; - } catch (const nlohmann::json::out_of_range &exception) { - spdlog::warn(get_failed_message("non existing field", exception.what())); - return; - } - MG_ASSERT(status.name == stream_name, "Expected stream name is '{}', but got '{}'", status.name, stream_name); - - try { - auto it = CreateConsumer(*locked_streams_map, stream_name, std::move(status.info), std::move(status.owner)); - if (status.is_running) { - std::visit( - [&](const auto &stream_data) { - auto stream_source_ptr = stream_data.stream_source->Lock(); - stream_source_ptr->Start(); - }, - it->second); - } - spdlog::info("Stream '{}' is loaded", stream_name); - } catch (const utils::BasicException &exception) { - spdlog::warn(get_failed_message("unexpected error", exception.what())); - } - }; - - auto stream_json_data = nlohmann::json::parse(stream_data); - if (const auto it = stream_json_data.find(kType); it != stream_json_data.end()) { - const auto stream_type = static_cast(*it); - switch (stream_type) { - case StreamSourceType::KAFKA: - create_consumer(StreamStatus{}, std::move(stream_json_data)); - break; - case StreamSourceType::PULSAR: - create_consumer(StreamStatus{}, std::move(stream_json_data)); - break; - } - } else { - spdlog::warn( - "Unable to load stream '{}', because it does not contain the type of the stream. Most probably the stream " - "was saved before Memgraph 2.1. Please recreate the stream manually to make it work. For more information " - "please check https://memgraph.com/docs/memgraph/changelog#v210---nov-22-2021 .", - stream_json_data.value(kStreamName, "")); - } - } -} - -void Streams::Drop(const std::string &stream_name) { - auto locked_streams = streams_.Lock(); - - auto it = GetStream(*locked_streams, stream_name); - - // streams_ is write locked, which means there is no access to it outside of this function, thus only the Test - // function can be executing with the consumer, nothing else. - // By acquiring the write lock here for the consumer, we make sure there is - // no running Test function for this consumer, therefore it can be erased. - std::visit([&](const auto &stream_data) { stream_data.stream_source->Lock(); }, it->second); - - locked_streams->erase(it); - if (!storage_.Delete(stream_name)) { - throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name); - } - - // TODO(antaljanosbenjamin) Release the transformation -} - -void Streams::Start(const std::string &stream_name) { - auto locked_streams = streams_.Lock(); - auto it = GetStream(*locked_streams, stream_name); - - std::visit( - [&, this](const auto &stream_data) { - auto stream_source_ptr = stream_data.stream_source->Lock(); - stream_source_ptr->Start(); - Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr)); - }, - it->second); -} - -void Streams::StartWithLimit(const std::string &stream_name, uint64_t batch_limit, - std::optional timeout) const { - std::optional locked_streams{streams_.ReadLock()}; - auto it = GetStream(**locked_streams, stream_name); - - std::visit( - [&](const auto &stream_data) { - const auto locked_stream_source = stream_data.stream_source->ReadLock(); - locked_streams.reset(); - - locked_stream_source->StartWithLimit(batch_limit, timeout); - }, - it->second); -} - -void Streams::Stop(const std::string &stream_name) { - auto locked_streams = streams_.Lock(); - auto it = GetStream(*locked_streams, stream_name); - - std::visit( - [&, this](const auto &stream_data) { - auto stream_source_ptr = stream_data.stream_source->Lock(); - stream_source_ptr->Stop(); - - Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr)); - }, - it->second); -} - -void Streams::StartAll() { - for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) { - std::visit( - [&stream_name = stream_name, this](const auto &stream_data) { - auto locked_stream_source = stream_data.stream_source->Lock(); - if (!locked_stream_source->IsRunning()) { - locked_stream_source->Start(); - Persist( - CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source)); - } - }, - stream_data); - } -} - -void Streams::StopAll() { - for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) { - std::visit( - [&stream_name = stream_name, this](const auto &stream_data) { - auto locked_stream_source = stream_data.stream_source->Lock(); - if (locked_stream_source->IsRunning()) { - locked_stream_source->Stop(); - Persist( - CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source)); - } - }, - stream_data); - } -} - -std::vector> Streams::GetStreamInfo() const { - std::vector> result; - { - for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) { - std::visit( - [&, &stream_name = stream_name](const auto &stream_data) { - auto locked_stream_source = stream_data.stream_source->ReadLock(); - auto info = locked_stream_source->Info(stream_data.transformation_name); - result.emplace_back(StreamStatus<>{stream_name, StreamType(*locked_stream_source), - locked_stream_source->IsRunning(), std::move(info.common_info), - stream_data.owner}); - }, - stream_data); - } - } - return result; -} - -TransformationResult Streams::Check(const std::string &stream_name, std::optional timeout, - std::optional batch_limit) const { - std::optional locked_streams{streams_.ReadLock()}; - auto it = GetStream(**locked_streams, stream_name); - - return std::visit( - [&](const auto &stream_data) { - // This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after - // that - const auto locked_stream_source = stream_data.stream_source->ReadLock(); - const auto transformation_name = stream_data.transformation_name; - locked_streams.reset(); - - auto *memory_resource = utils::NewDeleteResource(); - mgp_result result{nullptr, memory_resource}; - TransformationResult test_result; - - auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name, - &transformation_name = transformation_name, &result, - &test_result](const std::vector &messages) mutable { - auto accessor = interpreter_context->db->Access(coordinator::Hlc{}); - CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name); - - auto result_row = std::vector(); - result_row.reserve(kCheckStreamResultSize); - - auto queries_and_parameters = std::vector(result.rows.size()); - std::transform( - result.rows.cbegin(), result.rows.cend(), queries_and_parameters.begin(), [&](const auto &row) { - auto [query, parameters] = ExtractTransformationResult(row.values, transformation_name, stream_name); - - return std::map{{"query", std::move(query)}, - {"parameters", std::move(parameters)}}; - }); - result_row.emplace_back(std::move(queries_and_parameters)); - - auto messages_list = std::vector(messages.size()); - std::transform(messages.cbegin(), messages.cend(), messages_list.begin(), [](const auto &message) { - return std::string_view(message.Payload().data(), message.Payload().size()); - }); - - result_row.emplace_back(std::move(messages_list)); - - test_result.emplace_back(std::move(result_row)); - }; - - locked_stream_source->Check(timeout, batch_limit, consumer_function); - return test_result; - }, - it->second); -} - -} // namespace memgraph::query::v2::stream diff --git a/src/query/v2/stream/streams.hpp b/src/query/v2/stream/streams.hpp deleted file mode 100644 index 1d4451978..000000000 --- a/src/query/v2/stream/streams.hpp +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "integrations/kafka/consumer.hpp" -#include "kvstore/kvstore.hpp" -#include "query/v2/bindings/typed_value.hpp" -#include "query/v2/stream/common.hpp" -#include "query/v2/stream/sources.hpp" -#include "storage/v3/property_value.hpp" -#include "utils/event_counter.hpp" -#include "utils/exceptions.hpp" -#include "utils/rw_lock.hpp" -#include "utils/synchronized.hpp" - -class StreamsTest; -namespace memgraph::query::v2 { - -struct InterpreterContext; - -namespace stream { - -class StreamsException : public utils::BasicException { - public: - using BasicException::BasicException; -}; - -template -struct StreamInfo; - -template <> -struct StreamInfo { - using Type = CommonStreamInfo; -}; - -template -struct StreamInfo { - using Type = typename TStream::StreamInfo; -}; - -template -using StreamInfoType = typename StreamInfo::Type; - -template -struct StreamStatus { - std::string name; - StreamSourceType type; - bool is_running; - StreamInfoType info; - std::optional owner; -}; - -using TransformationResult = std::vector>; - -/// Manages Kafka consumers. -/// -/// This class is responsible for all query supported actions to happen. -class Streams final { - friend StreamsTest; - - public: - /// Initializes the streams. - /// - /// @param interpreter_context context to use to run the result of transformations - /// @param directory a directory path to store the persisted streams metadata - Streams(InterpreterContext *interpreter_context, std::filesystem::path directory); - - /// Restores the streams from the persisted metadata. - /// The restoration is done in a best effort manner, therefore no exception is thrown on failure, but the error is - /// logged. If a stream was running previously, then after restoration it will be started. - /// This function should only be called when there are no existing streams. - void RestoreStreams(); - - /// Creates a new import stream. - /// The create implies connecting to the server to get metadata necessary to initialize the stream. This - /// method assures there is no other stream with the same name. - /// - /// @param stream_name the name of the stream which can be used to uniquely identify the stream - /// @param stream_info the necessary informations needed to create the Kafka consumer and transform the messages - /// - /// @throws StreamsException if the stream with the same name exists or if the creation of Kafka consumer fails - template - void Create(const std::string &stream_name, typename TStream::StreamInfo info, std::optional owner); - - /// Deletes an existing stream and all the data that was persisted. - /// - /// @param stream_name name of the stream that needs to be deleted. - /// - /// @throws StreamsException if the stream doesn't exist or if the persisted metadata can't be deleted. - void Drop(const std::string &stream_name); - - /// Start consuming from a stream. - /// - /// @param stream_name name of the stream that needs to be started - /// - /// @throws StreamsException if the stream doesn't exist or if the metadata cannot be persisted - /// @throws ConsumerRunningException if the consumer is already running - void Start(const std::string &stream_name); - - /// Start consuming from a stream. - /// - /// @param stream_name name of the stream that needs to be started - /// @param batch_limit number of batches we want to consume before stopping - /// @param timeout the maximum duration during which the command should run. - /// - /// @throws StreamsException if the stream doesn't exist - /// @throws ConsumerRunningException if the consumer is already running - void StartWithLimit(const std::string &stream_name, uint64_t batch_limit, - std::optional timeout) const; - - /// Stop consuming from a stream. - /// - /// @param stream_name name of the stream that needs to be stopped - /// - /// @throws StreamsException if the stream doesn't exist or if the metadata cannot be persisted - /// @throws ConsumerStoppedException if the consumer is already stopped - void Stop(const std::string &stream_name); - - /// Start consuming from all streams that are stopped. - /// - /// @throws StreamsException if the metadata cannot be persisted - void StartAll(); - - /// Stop consuming from all streams that are running. - /// - /// @throws StreamsException if the metadata cannot be persisted - void StopAll(); - - /// Return current status for all streams. - /// It might happend that the is_running field is out of date if the one of the streams stops during the invocation of - /// this function because of an error. - std::vector> GetStreamInfo() const; - - /// Do a dry-run consume from a stream. - /// - /// @param stream_name name of the stream we want to test - /// @param batch_limit number of batches we want to test before stopping - /// @param timeout the maximum duration during which the command should run. - /// - /// @returns A vector of vectors of TypedValue. Each subvector contains two elements, the query string and the - /// nullable parameters map. - /// - /// @throws StreamsException if the stream doesn't exist - /// @throws ConsumerRunningException if the consumer is alredy running - /// @throws ConsumerCheckFailedException if the transformation function throws any std::exception during processing - TransformationResult Check(const std::string &stream_name, - std::optional timeout = std::nullopt, - std::optional batch_limit = std::nullopt) const; - - private: - template - using SynchronizedStreamSource = utils::Synchronized; - - template - struct StreamData { - std::string transformation_name; - std::optional owner; - std::unique_ptr> stream_source; - }; - - using StreamDataVariant = std::variant, StreamData>; - using StreamsMap = std::unordered_map; - using SynchronizedStreamsMap = utils::Synchronized; - - template - StreamsMap::iterator CreateConsumer(StreamsMap &map, const std::string &stream_name, - typename TStream::StreamInfo stream_info, std::optional owner); - - template - void Persist(StreamStatus &&status) { - const std::string stream_name = status.name; - if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) { - throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name}; - } - } - - void RegisterProcedures(); - void RegisterKafkaProcedures(); - void RegisterPulsarProcedures(); - - InterpreterContext *interpreter_context_; - kvstore::KVStore storage_; - - SynchronizedStreamsMap streams_; -}; - -} // namespace stream -} // namespace memgraph::query::v2 diff --git a/src/query/v2/trigger.cpp b/src/query/v2/trigger.cpp index f2135f913..008b869f8 100644 --- a/src/query/v2/trigger.cpp +++ b/src/query/v2/trigger.cpp @@ -183,7 +183,7 @@ std::shared_ptr Trigger::GetPlan(DbAccessor *db_accessor, [](auto &identifier) { return &identifier.first; }); auto logical_plan = MakeLogicalPlan(std::move(ast_storage), utils::Downcast(parsed_statements_.query), - parsed_statements_.parameters, db_accessor, predefined_identifiers); + parsed_statements_.parameters, nullptr, predefined_identifiers); trigger_plan_ = std::make_shared(std::move(logical_plan), std::move(identifiers)); } @@ -210,8 +210,8 @@ void Trigger::Execute(DbAccessor *dba, utils::MonotonicBufferResource *execution ctx.symbol_table = plan.symbol_table(); ctx.evaluation_context.timestamp = QueryTimestamp(); ctx.evaluation_context.parameters = parsed_statements_.parameters; - ctx.evaluation_context.properties = NamesToProperties(plan.ast_storage().properties_, dba); - ctx.evaluation_context.labels = NamesToLabels(plan.ast_storage().labels_, dba); + ctx.evaluation_context.properties = NamesToProperties(plan.ast_storage().properties_, nullptr); + ctx.evaluation_context.labels = NamesToLabels(plan.ast_storage().labels_, nullptr); ctx.timer = utils::AsyncTimer(max_execution_time_sec); ctx.is_shutting_down = is_shutting_down; ctx.is_profile_query = false; diff --git a/src/query/v2/trigger_context.hpp b/src/query/v2/trigger_context.hpp index 4665cf1d5..0b7dac01f 100644 --- a/src/query/v2/trigger_context.hpp +++ b/src/query/v2/trigger_context.hpp @@ -77,7 +77,7 @@ struct SetObjectProperty { std::map ToMap(DbAccessor *dba) const { return {{ObjectString(), TypedValue{object}}, - {"key", TypedValue{dba->PropertyToName(key)}}, + {"key", TypedValue{1}}, // TODO Fix trigger {"old", old_value}, {"new", new_value}}; } @@ -97,7 +97,7 @@ struct RemovedObjectProperty { std::map ToMap(DbAccessor *dba) const { return {{ObjectString(), TypedValue{object}}, - {"key", TypedValue{dba->PropertyToName(key)}}, + {"key", TypedValue{1}}, // TODO Fix triggers {"old", old_value}}; } diff --git a/src/storage/v3/bindings/db_accessor.hpp b/src/storage/v3/bindings/db_accessor.hpp index 28d603e3d..1424c1542 100644 --- a/src/storage/v3/bindings/db_accessor.hpp +++ b/src/storage/v3/bindings/db_accessor.hpp @@ -11,6 +11,7 @@ #pragma once +#include "storage/v3/result.hpp" #include "storage/v3/shard.hpp" namespace memgraph::storage::v3 { @@ -88,7 +89,7 @@ class DbAccessor final { if (maybe_vertex_acc.HasError()) { return {std::move(maybe_vertex_acc.GetError())}; } - return VertexAccessor{maybe_vertex_acc.GetValue()}; + return maybe_vertex_acc.GetValue(); } storage::v3::Result InsertEdge(VertexAccessor *from, VertexAccessor *to, diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 083455a25..4428cd10c 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -340,6 +340,7 @@ Shard::~Shard() {} Shard::Accessor::Accessor(Shard &shard, Transaction &transaction) : shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {} +// TODO(jbajic) Remove with next PR ResultSchema Shard::Accessor::CreateVertexAndValidate( LabelId primary_label, const std::vector &labels, const std::vector> &properties) { @@ -387,16 +388,23 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( } ResultSchema Shard::Accessor::CreateVertexAndValidate( - LabelId primary_label, const std::vector &labels, const std::vector &primary_properties, + const std::vector &labels, const std::vector &primary_properties, const std::vector> &properties) { - if (primary_label != shard_->primary_label_) { - throw utils::BasicException("Cannot add vertex to shard which does not hold the given primary label!"); + OOMExceptionEnabler oom_exception; + const auto schema = shard_->GetSchema(shard_->primary_label_)->second; + std::vector> primary_properties_ordered; + // TODO(jbajic) Maybe react immediately and send Violation + MG_ASSERT("PrimaryKey is invalid size"); + for (auto i{0}; i < schema.size(); ++i) { + primary_properties_ordered.emplace_back(schema[i].property_id, primary_properties[i]); } - auto maybe_schema_violation = GetSchemaValidator().ValidateVertexCreate(primary_label, labels, properties); + + auto maybe_schema_violation = + GetSchemaValidator().ValidateVertexCreate(shard_->primary_label_, labels, primary_properties_ordered); if (maybe_schema_violation) { return {std::move(*maybe_schema_violation)}; } - OOMExceptionEnabler oom_exception; + auto acc = shard_->vertices_.access(); auto *delta = CreateDeleteObjectDelta(transaction_); auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}}); @@ -408,7 +416,7 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( // TODO(jbajic) Improve, maybe delay index update for (const auto &[property_id, property_value] : properties) { - if (!shard_->schemas_.IsPropertyKey(primary_label, property_id)) { + if (!shard_->schemas_.IsPropertyKey(shard_->primary_label_, property_id)) { if (const auto err = vertex_acc.SetProperty(property_id, property_value); err.HasError()) { return {err.GetError()}; } @@ -697,6 +705,8 @@ const std::string &Shard::Accessor::EdgeTypeToName(EdgeTypeId edge_type) const { return shard_->EdgeTypeToName(edge_type); } +LabelId Shard::PrimaryLabel() const { return primary_label_; } + void Shard::Accessor::AdvanceCommand() { ++transaction_->command_id; } void Shard::Accessor::Commit(coordinator::Hlc commit_timestamp) { diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 4b0b9d93c..db54741da 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -208,14 +208,13 @@ class Shard final { // TODO(gvolfing) this is just a workaround for stitching remove this later. LabelId GetPrimaryLabel() const noexcept { return shard_->primary_label_; } - /// @throw std::bad_alloc ResultSchema CreateVertexAndValidate( LabelId primary_label, const std::vector &labels, const std::vector> &properties); /// @throw std::bad_alloc ResultSchema CreateVertexAndValidate( - LabelId primary_label, const std::vector &labels, const std::vector &primary_properties, + const std::vector &labels, const std::vector &primary_properties, const std::vector> &properties); std::optional FindVertex(std::vector primary_key, View view); @@ -341,6 +340,8 @@ class Shard final { const std::string &EdgeTypeToName(EdgeTypeId edge_type) const; + LabelId PrimaryLabel() const; + /// @throw std::bad_alloc bool CreateIndex(LabelId label, std::optional desired_commit_timestamp = {}); diff --git a/src/storage/v3/shard_manager.hpp b/src/storage/v3/shard_manager.hpp index e682646e0..aa9f938e8 100644 --- a/src/storage/v3/shard_manager.hpp +++ b/src/storage/v3/shard_manager.hpp @@ -26,6 +26,7 @@ #include #include #include +#include "coordinator/shard_map.hpp" #include "storage/v3/config.hpp" namespace memgraph::storage::v3 { @@ -76,7 +77,8 @@ static_assert(kMinimumCronInterval < kMaximumCronInterval, template class ShardManager { public: - ShardManager(io::Io io, Address coordinator_leader) : io_(io), coordinator_leader_(coordinator_leader) {} + ShardManager(io::Io io, Address coordinator_leader, coordinator::ShardMap shard_map) + : io_(io), coordinator_leader_(coordinator_leader), shard_map_{std::move(shard_map)} {} /// Periodic protocol maintenance. Returns the time that Cron should be called again /// in the future. @@ -135,6 +137,7 @@ class ShardManager { std::priority_queue, std::vector>, std::greater<>> cron_schedule_; Time next_cron_; Address coordinator_leader_; + coordinator::ShardMap shard_map_; std::optional>> heartbeat_res_; // TODO(tyler) over time remove items from initialized_but_not_confirmed_rsm_ @@ -212,6 +215,17 @@ class ShardManager { std::unique_ptr shard = std::make_unique(to_init.label_id, to_init.min_key, to_init.max_key, to_init.schema, to_init.config); + // TODO(jbajic) Should be sync with coordinator and not passed + std::unordered_map id_to_name; + const auto map_type_ids = [&id_to_name](const auto &name_to_id_type) { + for (const auto &[name, id] : name_to_id_type) { + id_to_name.insert({id.AsUint(), name}); + } + }; + map_type_ids(shard_map_.edge_types); + map_type_ids(shard_map_.labels); + map_type_ids(shard_map_.properties); + shard->StoreMapping(std::move(id_to_name)); ShardRsm rsm_state{std::move(shard)}; diff --git a/src/storage/v3/shard_rsm.cpp b/src/storage/v3/shard_rsm.cpp index 4a975529d..1c161b890 100644 --- a/src/storage/v3/shard_rsm.cpp +++ b/src/storage/v3/shard_rsm.cpp @@ -15,6 +15,9 @@ #include "parser/opencypher/parser.hpp" #include "query/v2/requests.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/schemas.hpp" #include "storage/v3/shard_rsm.hpp" #include "storage/v3/value_conversions.hpp" #include "storage/v3/vertex_accessor.hpp" @@ -78,18 +81,29 @@ std::optional> CollectSpecificPropertiesFromAccessor } std::optional> CollectAllPropertiesFromAccessor( - const memgraph::storage::v3::VertexAccessor &acc, memgraph::storage::v3::View view) { + const memgraph::storage::v3::VertexAccessor &acc, memgraph::storage::v3::View view, + const memgraph::storage::v3::Schemas::Schema *schema) { std::map ret; - auto iter = acc.Properties(view); - if (iter.HasError()) { + auto props = acc.Properties(view); + if (props.HasError()) { spdlog::debug("Encountered an error while trying to get vertex properties."); return std::nullopt; } - - for (const auto &[prop_key, prop_val] : iter.GetValue()) { + for (const auto &[prop_key, prop_val] : props.GetValue()) { ret.emplace(prop_key, ToValue(prop_val)); } + auto maybe_pk = acc.PrimaryKey(view); + if (maybe_pk.HasError()) { + spdlog::debug("Encountered an error while trying to get vertex primary key."); + } + + const auto pk = maybe_pk.GetValue(); + MG_ASSERT(schema->second.size() == pk.size(), "PrimaryKey size does not match schema!"); + for (size_t i{0}; i < schema->second.size(); ++i) { + ret.emplace(schema->second[i].property_id, ToValue(pk[i])); + } + return ret; } @@ -406,10 +420,6 @@ namespace memgraph::storage::v3 { msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) { auto acc = shard_->Access(req.transaction_id); - // Workaround untill we have access to CreateVertexAndValidate() - // with the new signature that does not require the primary label. - const auto prim_label = acc.GetPrimaryLabel(); - bool action_successful = true; for (auto &new_vertex : req.new_vertices) { @@ -428,10 +438,12 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) { for (const auto &label_id : new_vertex.label_ids) { converted_label_ids.emplace_back(label_id.id); } - - auto result_schema = - acc.CreateVertexAndValidate(prim_label, converted_label_ids, - ConvertPropertyVector(std::move(new_vertex.primary_key)), converted_property_map); + // TODO(jbajic) sending primary key as vector breaks validation on storage side + // cannot map id -> value + PrimaryKey transformed_pk; + std::transform(new_vertex.primary_key.begin(), new_vertex.primary_key.end(), std::back_inserter(transformed_pk), + [](const auto &val) { return ToPropertyValue(val); }); + auto result_schema = acc.CreateVertexAndValidate(converted_label_ids, transformed_pk, converted_property_map); if (result_schema.HasError()) { auto &error = result_schema.GetError(); @@ -695,17 +707,18 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) { for (auto it = vertex_iterable.begin(); it != vertex_iterable.end(); ++it) { const auto &vertex = *it; - if (start_ids == vertex.PrimaryKey(View(req.storage_view)).GetValue()) { + if (start_ids <= vertex.PrimaryKey(View(req.storage_view)).GetValue()) { did_reach_starting_point = true; } if (did_reach_starting_point) { std::optional> found_props; + const auto *schema = shard_->GetSchema(shard_->PrimaryLabel()); if (req.props_to_return) { found_props = CollectSpecificPropertiesFromAccessor(vertex, req.props_to_return.value(), view); } else { - found_props = CollectAllPropertiesFromAccessor(vertex, view); + found_props = CollectAllPropertiesFromAccessor(vertex, view, schema); } // TODO(gvolfing) -VERIFY- @@ -720,7 +733,7 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) { .props = FromMap(found_props.value())}); ++sample_counter; - if (sample_counter == req.batch_limit) { + if (req.batch_limit && sample_counter == req.batch_limit) { // Reached the maximum specified batch size. // Get the next element before exiting. const auto &next_vertex = *(++it); diff --git a/tests/unit/machine_manager.cpp b/tests/unit/machine_manager.cpp index 118e9d34b..8c9a2216a 100644 --- a/tests/unit/machine_manager.cpp +++ b/tests/unit/machine_manager.cpp @@ -151,7 +151,7 @@ MachineManager MkMm(LocalSystem &local_system, std::vector mm) { mm.Run(); } diff --git a/tests/unit/query_v2_dummy_test.cpp b/tests/unit/query_v2_dummy_test.cpp index 302de4b3a..42c06b9ec 100644 --- a/tests/unit/query_v2_dummy_test.cpp +++ b/tests/unit/query_v2_dummy_test.cpp @@ -26,7 +26,6 @@ #include "query/v2/plan/operator.hpp" #include "query/v2/plan/operator.hpp" -#include "query_v2_query_plan_common.hpp" class Dummy : public testing::Test { protected: