From 814c5eb397be925fbeea2742d6bb1251f71b4465 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Wed, 23 Nov 2022 15:15:26 +0100 Subject: [PATCH 001/251] Add the possibility of primary-key based indexing Add new possibility to base our indexing solution on. Add ScanAllOperator that represents the semantics and integrate its use through index_lookup. --- src/query/v2/plan/operator.cpp | 15 ++++++++++++++ src/query/v2/plan/operator.lcp | 24 +++++++++++++++++++++- src/query/v2/plan/rewrite/index_lookup.hpp | 22 ++++++++++++++++++++ src/query/v2/plan/vertex_count_cache.hpp | 24 +++++++++++++++++++++- src/query/v2/shard_request_manager.hpp | 5 +++++ 5 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 068ca9192..196f405aa 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -92,6 +92,7 @@ extern const Event ScanAllByLabelPropertyRangeOperator; extern const Event ScanAllByLabelPropertyValueOperator; extern const Event ScanAllByLabelPropertyOperator; extern const Event ScanAllByIdOperator; +extern const Event ScanAllByPrimaryKeyOperator; extern const Event ExpandOperator; extern const Event ExpandVariableOperator; extern const Event ConstructNamedPathOperator; @@ -545,6 +546,20 @@ UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) c throw QueryRuntimeException("ScanAllByLabelProperty is not supported"); } +ScanAllByPrimaryKey::ScanAllByPrimaryKey(const std::shared_ptr &input, Symbol output_symbol, + storage::v3::LabelId label, std::vector primary_key, + storage::v3::View view) + : ScanAll(input, output_symbol, view), label_(label), primary_key_(primary_key) { + MG_ASSERT(primary_key.front()); +} + +ACCEPT_WITH_INPUT(ScanAllByPrimaryKey) + +UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) const { + // EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); + throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); +} + ScanAllById::ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, storage::v3::View view) : ScanAll(input, output_symbol, view), expression_(expression) { diff --git a/src/query/v2/plan/operator.lcp b/src/query/v2/plan/operator.lcp index d6277809c..680bb4de3 100644 --- a/src/query/v2/plan/operator.lcp +++ b/src/query/v2/plan/operator.lcp @@ -111,6 +111,7 @@ class ScanAllByLabelPropertyRange; class ScanAllByLabelPropertyValue; class ScanAllByLabelProperty; class ScanAllById; +class ScanAllByPrimaryKey; class Expand; class ExpandVariable; class ConstructNamedPath; @@ -141,7 +142,7 @@ class Foreach; using LogicalOperatorCompositeVisitor = utils::CompositeVisitor< Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel, ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue, - ScanAllByLabelProperty, ScanAllById, + ScanAllByLabelProperty, ScanAllById, ScanAllByPrimaryKey, Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete, SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge, @@ -841,7 +842,28 @@ given label and property. (:serialize (:slk)) (:clone)) +(lcp:define-class scan-all-by-primary-key (scan-all) + ((label "::storage::v3::LabelId" :scope :public) + (primary-key "std::vector" :scope :public) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "ScanAll producing a single node with specified by the label and primary key") + (:public + #>cpp + ScanAllByPrimaryKey() {} + ScanAllByPrimaryKey(const std::shared_ptr &input, + Symbol output_symbol, + storage::v3::LabelId label, + std::vector primary_key, + storage::v3::View view = storage::v3::View::OLD); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) (lcp:define-class scan-all-by-id (scan-all) ((expression "Expression *" :scope :public diff --git a/src/query/v2/plan/rewrite/index_lookup.hpp b/src/query/v2/plan/rewrite/index_lookup.hpp index 57ddba54e..a861153e9 100644 --- a/src/query/v2/plan/rewrite/index_lookup.hpp +++ b/src/query/v2/plan/rewrite/index_lookup.hpp @@ -25,8 +25,10 @@ #include +#include "query/v2/frontend/ast/ast.hpp" #include "query/v2/plan/operator.hpp" #include "query/v2/plan/preprocess.hpp" +#include "storage/v3/id_types.hpp" DECLARE_int64(query_vertex_count_to_expand_existing); @@ -584,6 +586,26 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { // Without labels, we cannot generate any indexed ScanAll. return nullptr; } + + // First, try to see if we can find a vertex based on the possibly + // supplied primary key. + auto property_filters = filters_.PropertyFilters(node_symbol); + storage::v3::LabelId prim_label; + std::vector primary_key; + + if (!property_filters.empty()) { + for (const auto &label : labels) { + if (db_->LabelIndexExists(GetLabel(label))) { + prim_label = GetLabel(label); + primary_key = db_->ExtractPrimaryKey(prim_label, property_filters); + break; + } + } + if (!primary_key.empty()) { + return std::make_unique(input, node_symbol, prim_label, primary_key); + } + } + auto found_index = FindBestLabelPropertyIndex(node_symbol, bound_symbols); if (found_index && // Use label+property index if we satisfy max_vertex_count. diff --git a/src/query/v2/plan/vertex_count_cache.hpp b/src/query/v2/plan/vertex_count_cache.hpp index a7bfbdf85..70b8d324b 100644 --- a/src/query/v2/plan/vertex_count_cache.hpp +++ b/src/query/v2/plan/vertex_count_cache.hpp @@ -12,9 +12,11 @@ /// @file #pragma once +#include #include #include "query/v2/bindings/typed_value.hpp" +#include "query/v2/plan/preprocess.hpp" #include "query/v2/shard_request_manager.hpp" #include "storage/v3/conversions.hpp" #include "storage/v3/id_types.hpp" @@ -52,11 +54,31 @@ class VertexCountCache { return 1; } - // For now return true if label is primary label bool LabelIndexExists(storage::v3::LabelId label) { return shard_request_manager_->IsPrimaryLabel(label); } bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; } + std::vector ExtractPrimaryKey(storage::v3::LabelId label, + std::vector property_filters) { + std::vector pk; + const auto schema = shard_request_manager_->GetSchemaForLabel(label); + + std::vector schema_properties; + schema_properties.reserve(schema.size()); + + std::transform(schema.begin(), schema.end(), std::back_inserter(schema_properties), + [](const auto &schema_elem) { return schema_elem.property_id; }); + + for (const auto &property_filter : property_filters) { + const auto &property_id = NameToProperty(property_filter.property_filter->property_.name); + if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) { + pk.push_back(property_filter.expression); + } + } + + return pk.size() == schema_properties.size() ? pk : std::vector{}; + } + msgs::ShardRequestManagerInterface *shard_request_manager_; }; diff --git a/src/query/v2/shard_request_manager.hpp b/src/query/v2/shard_request_manager.hpp index a73201046..44ea02bba 100644 --- a/src/query/v2/shard_request_manager.hpp +++ b/src/query/v2/shard_request_manager.hpp @@ -131,6 +131,7 @@ class ShardRequestManagerInterface { virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0; virtual bool IsPrimaryLabel(LabelId label) const = 0; virtual bool IsPrimaryKey(LabelId primary_label, PropertyId property) const = 0; + virtual std::vector GetSchemaForLabel(LabelId label) const = 0; }; // TODO(kostasrim)rename this class template @@ -244,6 +245,10 @@ class ShardRequestManager : public ShardRequestManagerInterface { }) != schema_it->second.end(); } + std::vector GetSchemaForLabel(LabelId label) const override { + return shards_map_.schemas.at(label); + } + bool IsPrimaryLabel(LabelId label) const override { return shards_map_.label_spaces.contains(label); } // TODO(kostasrim) Simplify return result From 1b73ca4860c4e717596be181d3fd7f9b6b501db4 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Wed, 23 Nov 2022 15:49:59 +0100 Subject: [PATCH 002/251] Remove ScanAllById operator --- src/query/v2/plan/cost_estimator.hpp | 2 -- src/query/v2/plan/operator.cpp | 19 ------------ src/query/v2/plan/operator.lcp | 21 +------------ src/query/v2/plan/pretty_print.cpp | 18 ----------- src/query/v2/plan/pretty_print.hpp | 2 -- src/query/v2/plan/read_write_type_checker.cpp | 1 - src/query/v2/plan/read_write_type_checker.hpp | 1 - src/query/v2/plan/rewrite/index_lookup.hpp | 30 ++----------------- 8 files changed, 3 insertions(+), 91 deletions(-) diff --git a/src/query/v2/plan/cost_estimator.hpp b/src/query/v2/plan/cost_estimator.hpp index 8cba2505f..250274a73 100644 --- a/src/query/v2/plan/cost_estimator.hpp +++ b/src/query/v2/plan/cost_estimator.hpp @@ -149,8 +149,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { return true; } - // TODO: Cost estimate ScanAllById? - // For the given op first increments the cardinality and then cost. #define POST_VISIT_CARD_FIRST(NAME) \ bool PostVisit(NAME &) override { \ diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 196f405aa..29bf07792 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -91,7 +91,6 @@ extern const Event ScanAllByLabelOperator; extern const Event ScanAllByLabelPropertyRangeOperator; extern const Event ScanAllByLabelPropertyValueOperator; extern const Event ScanAllByLabelPropertyOperator; -extern const Event ScanAllByIdOperator; extern const Event ScanAllByPrimaryKeyOperator; extern const Event ExpandOperator; extern const Event ExpandVariableOperator; @@ -560,24 +559,6 @@ UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) cons throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); } -ScanAllById::ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, - storage::v3::View view) - : ScanAll(input, output_symbol, view), expression_(expression) { - MG_ASSERT(expression); -} - -ACCEPT_WITH_INPUT(ScanAllById) - -UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const { - EventCounter::IncrementCounter(EventCounter::ScanAllByIdOperator); - // TODO Reimplement when we have reliable conversion between hash value and pk - auto vertices = [](Frame & /*frame*/, ExecutionContext & /*context*/) -> std::optional> { - return std::nullopt; - }; - return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), - std::move(vertices), "ScanAllById"); -} - namespace { template diff --git a/src/query/v2/plan/operator.lcp b/src/query/v2/plan/operator.lcp index 680bb4de3..4a4df3785 100644 --- a/src/query/v2/plan/operator.lcp +++ b/src/query/v2/plan/operator.lcp @@ -110,7 +110,6 @@ class ScanAllByLabel; class ScanAllByLabelPropertyRange; class ScanAllByLabelPropertyValue; class ScanAllByLabelProperty; -class ScanAllById; class ScanAllByPrimaryKey; class Expand; class ExpandVariable; @@ -142,7 +141,7 @@ class Foreach; using LogicalOperatorCompositeVisitor = utils::CompositeVisitor< Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel, ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue, - ScanAllByLabelProperty, ScanAllById, ScanAllByPrimaryKey, + ScanAllByLabelProperty, ScanAllByPrimaryKey, Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete, SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge, @@ -865,24 +864,6 @@ given label and property. (:serialize (:slk)) (:clone)) -(lcp:define-class scan-all-by-id (scan-all) - ((expression "Expression *" :scope :public - :slk-save #'slk-save-ast-pointer - :slk-load (slk-load-ast-pointer "Expression"))) - (:documentation - "ScanAll producing a single node with ID equal to evaluated expression") - (:public - #>cpp - ScanAllById() {} - ScanAllById(const std::shared_ptr &input, - Symbol output_symbol, Expression *expression, - storage::v3::View view = storage::v3::View::OLD); - - bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; - UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; - cpp<#) - (:serialize (:slk)) - (:clone)) (lcp:define-struct expand-common () ( diff --git a/src/query/v2/plan/pretty_print.cpp b/src/query/v2/plan/pretty_print.cpp index b756a6299..76cc6ac26 100644 --- a/src/query/v2/plan/pretty_print.cpp +++ b/src/query/v2/plan/pretty_print.cpp @@ -86,14 +86,6 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) { return true; } -bool PlanPrinter::PreVisit(ScanAllById &op) { - WithPrintLn([&](auto &out) { - out << "* ScanAllById" - << " (" << op.output_symbol_.name() << ")"; - }); - return true; -} - bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) { WithPrintLn([&](auto &out) { *out_ << "* Expand (" << op.input_symbol_.name() << ")" @@ -488,16 +480,6 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) { return false; } -bool PlanToJsonVisitor::PreVisit(ScanAllById &op) { - json self; - self["name"] = "ScanAllById"; - self["output_symbol"] = ToJson(op.output_symbol_); - op.input_->Accept(*this); - self["input"] = PopOutput(); - output_ = std::move(self); - return false; -} - bool PlanToJsonVisitor::PreVisit(CreateNode &op) { json self; self["name"] = "CreateNode"; diff --git a/src/query/v2/plan/pretty_print.hpp b/src/query/v2/plan/pretty_print.hpp index c7442b196..66fa31556 100644 --- a/src/query/v2/plan/pretty_print.hpp +++ b/src/query/v2/plan/pretty_print.hpp @@ -68,7 +68,6 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelProperty &) override; - bool PreVisit(ScanAllById &) override; bool PreVisit(Expand &) override; bool PreVisit(ExpandVariable &) override; @@ -196,7 +195,6 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelProperty &) override; - bool PreVisit(ScanAllById &) override; bool PreVisit(Produce &) override; bool PreVisit(Accumulate &) override; diff --git a/src/query/v2/plan/read_write_type_checker.cpp b/src/query/v2/plan/read_write_type_checker.cpp index 6cc38cedf..ec1459f6a 100644 --- a/src/query/v2/plan/read_write_type_checker.cpp +++ b/src/query/v2/plan/read_write_type_checker.cpp @@ -35,7 +35,6 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true) PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true) PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true) PRE_VISIT(ScanAllByLabelProperty, RWType::R, true) -PRE_VISIT(ScanAllById, RWType::R, true) PRE_VISIT(Expand, RWType::R, true) PRE_VISIT(ExpandVariable, RWType::R, true) diff --git a/src/query/v2/plan/read_write_type_checker.hpp b/src/query/v2/plan/read_write_type_checker.hpp index a3c2f1a46..e5f429035 100644 --- a/src/query/v2/plan/read_write_type_checker.hpp +++ b/src/query/v2/plan/read_write_type_checker.hpp @@ -59,7 +59,6 @@ class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelProperty &) override; - bool PreVisit(ScanAllById &) override; bool PreVisit(Expand &) override; bool PreVisit(ExpandVariable &) override; diff --git a/src/query/v2/plan/rewrite/index_lookup.hpp b/src/query/v2/plan/rewrite/index_lookup.hpp index a861153e9..41e24d2b6 100644 --- a/src/query/v2/plan/rewrite/index_lookup.hpp +++ b/src/query/v2/plan/rewrite/index_lookup.hpp @@ -273,15 +273,6 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { return true; } - bool PreVisit(ScanAllById &op) override { - prev_ops_.push_back(&op); - return true; - } - bool PostVisit(ScanAllById &) override { - prev_ops_.pop_back(); - return true; - } - bool PreVisit(ConstructNamedPath &op) override { prev_ops_.push_back(&op); return true; @@ -561,25 +552,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { const auto &view = scan.view_; const auto &modified_symbols = scan.ModifiedSymbols(*symbol_table_); std::unordered_set bound_symbols(modified_symbols.begin(), modified_symbols.end()); - auto are_bound = [&bound_symbols](const auto &used_symbols) { - for (const auto &used_symbol : used_symbols) { - if (!utils::Contains(bound_symbols, used_symbol)) { - return false; - } - } - return true; - }; - // First, try to see if we can find a vertex by ID. - if (!max_vertex_count || *max_vertex_count >= 1) { - for (const auto &filter : filters_.IdFilters(node_symbol)) { - if (filter.id_filter->is_symbol_in_value_ || !are_bound(filter.used_symbols)) continue; - auto *value = filter.id_filter->value_; - filter_exprs_for_removal_.insert(filter.expression); - filters_.EraseFilter(filter); - return std::make_unique(input, node_symbol, value, view); - } - } - // Now try to see if we can use label+property index. If not, try to use + + // Try to see if we can use label+property index. If not, try to use // just the label index. const auto labels = filters_.FilteredLabels(node_symbol); if (labels.empty()) { From a65ea4fe01e7398eb66dd0045ef9fef305370713 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 24 Nov 2022 09:37:47 +0100 Subject: [PATCH 003/251] Conform clang-tidy --- src/query/v2/plan/operator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 29bf07792..c55c5cb86 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -554,7 +554,7 @@ ScanAllByPrimaryKey::ScanAllByPrimaryKey(const std::shared_ptr ACCEPT_WITH_INPUT(ScanAllByPrimaryKey) -UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) const { +UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource * /*mem*/) const { // EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); } From 22e3164e60b3c758402b0d62fb19e3c2825ba300 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 28 Nov 2022 16:18:28 +0100 Subject: [PATCH 004/251] Add missing license --- tests/unit/CMakeLists.txt | 3 + tests/unit/query_common.hpp | 255 ++++ tests/unit/query_plan_checker.hpp | 140 ++- tests/unit/query_plan_checker_v2.hpp | 559 +++++++++ tests/unit/query_plan_v2.cpp | 1664 ++++++++++++++++++++++++++ 5 files changed, 2620 insertions(+), 1 deletion(-) create mode 100644 tests/unit/query_plan_checker_v2.hpp create mode 100644 tests/unit/query_plan_v2.cpp diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 0e5824318..d9f9df875 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -93,6 +93,9 @@ target_link_libraries(${test_prefix}query_expression_evaluator mg-query) add_unit_test(query_plan.cpp) target_link_libraries(${test_prefix}query_plan mg-query) +add_unit_test(query_plan_v2.cpp) +target_link_libraries(${test_prefix}query_plan_v2 mg-query-v2) + add_unit_test(query_plan_accumulate_aggregate.cpp) target_link_libraries(${test_prefix}query_plan_accumulate_aggregate mg-query) diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index 903cacf12..4697124ef 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -46,6 +46,8 @@ #include "storage/v2/id_types.hpp" #include "utils/string.hpp" +#include "query/v2/frontend/ast/ast.hpp" + namespace memgraph::query { namespace test_common { @@ -81,12 +83,39 @@ std::string ToString(NamedExpression *expr) { struct OrderBy { std::vector expressions; }; + +// new stuff begin + +struct OrderByv2 { + std::vector expressions; +}; + +// new stuff end + struct Skip { Expression *expression = nullptr; }; + +// new stuff begin + +struct Skipv2 { + query::v2::Expression *expression = nullptr; +}; + +// new stuff end + struct Limit { Expression *expression = nullptr; }; + +// new stuff begin + +struct Limitv2 { + query::v2::Expression *expression = nullptr; +}; + +// new stuff end + struct OnMatch { std::vector set; }; @@ -153,6 +182,42 @@ auto GetPropertyLookup(AstStorage &storage, TDbAccessor &, Expression *expr, return storage.Create(expr, storage.GetPropertyIx(prop_pair.first)); } +// new stuff begin + +template +auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, const std::string &name, + memgraph::storage::v3::PropertyId property) { + return storage.Create(storage.Create(name), + storage.GetPropertyIx(dba.PropertyToName(property))); +} + +template +auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, + memgraph::query::v2::Expression *expr, memgraph::storage::v3::PropertyId property) { + return storage.Create(expr, storage.GetPropertyIx(dba.PropertyToName(property))); +} + +template +auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, + memgraph::query::v2::Expression *expr, const std::string &property) { + return storage.Create(expr, storage.GetPropertyIx(property)); +} + +template +auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &, const std::string &name, + const std::pair &prop_pair) { + return storage.Create(storage.Create(name), + storage.GetPropertyIx(prop_pair.first)); +} + +template +auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &, memgraph::query::v2::Expression *expr, + const std::pair &prop_pair) { + return storage.Create(expr, storage.GetPropertyIx(prop_pair.first)); +} + +// new stuff end + /// Create an EdgeAtom with given name, direction and edge_type. /// /// Name is used to create the Identifier which is assigned to the edge. @@ -211,6 +276,15 @@ auto GetNode(AstStorage &storage, const std::string &name, std::optional label = std::nullopt) { + auto node = storage.Create(storage.Create(name)); + if (label) node->labels_.emplace_back(storage.GetLabelIx(*label)); + return node; +} +// new stuff end + /// Create a Pattern with given atoms. auto GetPattern(AstStorage &storage, std::vector atoms) { auto pattern = storage.Create(); @@ -227,6 +301,26 @@ auto GetPattern(AstStorage &storage, const std::string &name, std::vector atoms) { + auto pattern = storage.Create(); + pattern->identifier_ = storage.Create(memgraph::utils::RandomString(20), false); + pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); + return pattern; +} + +/// Create a Pattern with given name and atoms. +auto GetPattern(memgraph::query::v2::AstStorage &storage, const std::string &name, + std::vector atoms) { + auto pattern = storage.Create(); + pattern->identifier_ = storage.Create(name, true); + pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); + return pattern; +} + +// new stuff end + /// This function fills an AST node which with given patterns. /// /// The function is most commonly used to create Match and Create clauses. @@ -236,6 +330,16 @@ auto GetWithPatterns(TWithPatterns *with_patterns, std::vector patter return with_patterns; } +// new stuff begin + +template +auto GetWithPatterns(TWithPatterns *with_patterns, std::vector patterns) { + with_patterns->patterns_.insert(with_patterns->patterns_.begin(), patterns.begin(), patterns.end()); + return with_patterns; +} + +// new stuff end + /// Create a query with given clauses. auto GetSingleQuery(SingleQuery *single_query, Clause *clause) { @@ -271,6 +375,45 @@ auto GetSingleQuery(SingleQuery *single_query, Clause *clause, T *...clauses) { return GetSingleQuery(single_query, clauses...); } +// new stuff begin + +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Clause *clause) { + single_query->clauses_.emplace_back(clause); + return single_query; +} +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Match *match, query::v2::Where *where) { + match->where_ = where; + single_query->clauses_.emplace_back(match); + return single_query; +} +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::With *with, query::v2::Where *where) { + with->where_ = where; + single_query->clauses_.emplace_back(with); + return single_query; +} +template +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Match *match, query::v2::Where *where, + T *...clauses) { + match->where_ = where; + single_query->clauses_.emplace_back(match); + return GetSingleQuery(single_query, clauses...); +} +template +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::With *with, query::v2::Where *where, + T *...clauses) { + with->where_ = where; + single_query->clauses_.emplace_back(with); + return GetSingleQuery(single_query, clauses...); +} + +template +auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Clause *clause, T *...clauses) { + single_query->clauses_.emplace_back(clause); + return GetSingleQuery(single_query, clauses...); +} + +// new stuff end + auto GetCypherUnion(CypherUnion *cypher_union, SingleQuery *single_query) { cypher_union->single_query_ = single_query; return cypher_union; @@ -290,6 +433,24 @@ auto GetQuery(AstStorage &storage, SingleQuery *single_query, T *...cypher_union return query; } +// new stuff begin + +auto GetQuery(query::v2::AstStorage &storage, query::v2::SingleQuery *single_query) { + auto *query = storage.Create(); + query->single_query_ = single_query; + return query; +} + +template +auto GetQuery(query::v2::AstStorage &storage, query::v2::SingleQuery *single_query, T *...cypher_unions) { + auto *query = storage.Create(); + query->single_query_ = single_query; + query->cypher_unions_ = std::vector{cypher_unions...}; + return query; +} + +// new stuff end + // Helper functions for constructing RETURN and WITH clauses. void FillReturnBody(AstStorage &, ReturnBody &body, NamedExpression *named_expr) { body.named_expressions.emplace_back(named_expr); @@ -353,6 +514,80 @@ void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &na FillReturnBody(storage, body, rest...); } +// new stuff begin + +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, query::v2::NamedExpression *named_expr) { + body.named_expressions.emplace_back(named_expr); +} +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name) { + if (name == "*") { + body.all_identifiers = true; + } else { + auto *ident = storage.Create(name); + auto *named_expr = storage.Create(name, ident); + body.named_expressions.emplace_back(named_expr); + } +} +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, Limitv2 limit) { + body.limit = limit.expression; +} +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, Skipv2 skip, Limitv2 limit = Limitv2{}) { + body.skip = skip.expression; + body.limit = limit.expression; +} +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, OrderByv2 order_by, + Limitv2 limit = Limitv2{}) { + body.order_by = order_by.expressions; + body.limit = limit.expression; +} +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, OrderByv2 order_by, Skipv2 skip, + Limitv2 limit = Limitv2{}) { + body.order_by = order_by.expressions; + body.skip = skip.expression; + body.limit = limit.expression; +} +void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, query::v2::Expression *expr, + query::v2::NamedExpression *named_expr) { + // This overload supports `RETURN(expr, AS(name))` construct, since + // NamedExpression does not inherit Expression. + named_expr->expression_ = expr; + body.named_expressions.emplace_back(named_expr); +} +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, + query::v2::NamedExpression *named_expr) { + named_expr->expression_ = storage.Create(name); + body.named_expressions.emplace_back(named_expr); +} +template +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, query::v2::Expression *expr, + query::v2::NamedExpression *named_expr, T... rest) { + named_expr->expression_ = expr; + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, query::v2::NamedExpression *named_expr, + T... rest) { + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, + query::v2::NamedExpression *named_expr, T... rest) { + named_expr->expression_ = storage.Create(name); + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, T... rest) { + auto *ident = storage.Create(name); + auto *named_expr = storage.Create(name, ident); + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} + +// new stuff end + /// Create the return clause with given expressions. /// /// The supported expression combination of arguments is: @@ -374,6 +609,18 @@ auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { return ret; } +// new stuff begin + +template +auto GetReturn(query::v2::AstStorage &storage, bool distinct, T... exprs) { + auto ret = storage.Create(); + ret->body_.distinct = distinct; + FillReturnBody(storage, ret->body_, exprs...); + return ret; +} + +// new stuff end + /// Create the with clause with given expressions. /// /// The supported expression combination is the same as for @c GetReturn. @@ -486,12 +733,16 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec #define EDGE(...) memgraph::query::test_common::GetEdge(storage, __VA_ARGS__) #define EDGE_VARIABLE(...) memgraph::query::test_common::GetEdgeVariable(storage, __VA_ARGS__) #define PATTERN(...) memgraph::query::test_common::GetPattern(storage, {__VA_ARGS__}) +#define PATTERN(...) memgraph::query::test_common::GetPattern(storage, {__VA_ARGS__}) #define NAMED_PATTERN(name, ...) memgraph::query::test_common::GetPattern(storage, name, {__VA_ARGS__}) #define OPTIONAL_MATCH(...) \ memgraph::query::test_common::GetWithPatterns(storage.Create(true), {__VA_ARGS__}) #define MATCH(...) \ memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) +#define MATCH_V2(...) \ + memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) #define WHERE(expr) storage.Create((expr)) +#define WHERE_V2(expr) storage.Create((expr)) #define CREATE(...) \ memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) #define IDENT(...) storage.Create(__VA_ARGS__) @@ -501,6 +752,8 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec storage.Create( \ std::unordered_map{__VA_ARGS__}) #define PROPERTY_PAIR(property_name) std::make_pair(property_name, dba.NameToProperty(property_name)) +#define PRIMARY_PROPERTY_PAIR(property_name) std::make_pair(property_name, dba.NameToPrimaryProperty(property_name)) +#define SECONDARY_PROPERTY_PAIR(property_name) std::make_pair(property_name, dba.NameToSecondaryProperty(property_name)) #define PROPERTY_LOOKUP(...) memgraph::query::test_common::GetPropertyLookup(storage, dba, __VA_ARGS__) #define PARAMETER_LOOKUP(token_position) storage.Create((token_position)) #define NEXPR(name, expr) storage.Create((name), (expr)) @@ -536,6 +789,8 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec std::vector{(property)}) #define QUERY(...) memgraph::query::test_common::GetQuery(storage, __VA_ARGS__) #define SINGLE_QUERY(...) memgraph::query::test_common::GetSingleQuery(storage.Create(), __VA_ARGS__) +#define SINGLE_QUERY_V2(...) \ + memgraph::query::test_common::GetSingleQuery(storage.Create(), __VA_ARGS__) #define UNION(...) memgraph::query::test_common::GetCypherUnion(storage.Create(true), __VA_ARGS__) #define UNION_ALL(...) memgraph::query::test_common::GetCypherUnion(storage.Create(false), __VA_ARGS__) #define FOREACH(...) memgraph::query::test_common::GetForeach(storage, __VA_ARGS__) diff --git a/tests/unit/query_plan_checker.hpp b/tests/unit/query_plan_checker.hpp index 335b6ab2b..d04264c24 100644 --- a/tests/unit/query_plan_checker.hpp +++ b/tests/unit/query_plan_checker.hpp @@ -17,6 +17,7 @@ #include "query/plan/operator.hpp" #include "query/plan/planner.hpp" #include "query/plan/preprocess.hpp" +#include "query/v2/plan/operator.hpp" namespace memgraph::query::plan { @@ -90,7 +91,7 @@ class PlanChecker : public virtual HierarchicalLogicalOperatorVisitor { } PRE_VISIT(Unwind); PRE_VISIT(Distinct); - + bool PreVisit(Foreach &op) override { CheckOp(op); return false; @@ -336,6 +337,39 @@ class ExpectScanAllByLabelProperty : public OpChecker { memgraph::storage::PropertyId property_; }; +class ExpectScanAllByPrimaryKey : public OpChecker { + public: + ExpectScanAllByPrimaryKey(memgraph::storage::v3::LabelId label, const std::vector &properties) + : label_(label), properties_(properties) {} + + void ExpectOp(v2::plan::ScanAllByPrimaryKey &scan_all, const SymbolTable &) override { + EXPECT_EQ(scan_all.label_, label_); + // EXPECT_EQ(scan_all.property_, property_); + + // TODO(gvolfing) maybe assert the size of the 2 vectors. + // TODO(gvolfing) maybe use some std alg if Expression lets us. + + bool primary_property_match = true; + for (const auto &expected_prop : properties_) { + bool has_match = false; + for (const auto &prop : scan_all.primary_key_) { + if (typeid(prop).hash_code() == typeid(expected_prop).hash_code()) { + has_match = true; + } + } + if (!has_match) { + primary_property_match = false; + } + } + + EXPECT_TRUE(primary_property_match); + } + + private: + memgraph::storage::v3::LabelId label_; + std::vector properties_; +}; + class ExpectCartesian : public OpChecker { public: ExpectCartesian(const std::list> &left, @@ -486,4 +520,108 @@ class FakeDbAccessor { std::vector> label_property_index_; }; +// class FakeDistributedDbAccessor { +// public: +// int64_t VerticesCount(memgraph::storage::v3::LabelId label) const { +// auto found = label_index_.find(label); +// if (found != label_index_.end()) return found->second; +// return 0; +// } + +// int64_t VerticesCount(memgraph::storage::v3::LabelId label, memgraph::storage::PropertyId property) const { +// for (auto &index : label_property_index_) { +// if (std::get<0>(index) == label && std::get<1>(index) == property) { +// return std::get<2>(index); +// } +// } +// return 0; +// } + +// bool LabelIndexExists(memgraph::storage::v3::LabelId label) const { +// return label_index_.find(label) != label_index_.end(); +// } + +// bool LabelPropertyIndexExists(memgraph::storage::v3::LabelId label, memgraph::storage::PropertyId property) const { +// for (auto &index : label_property_index_) { +// if (std::get<0>(index) == label && std::get<1>(index) == property) { +// return true; +// } +// } +// return false; +// } + +// void SetIndexCount(memgraph::storage::v3::LabelId label, int64_t count) { label_index_[label] = count; } + +// void SetIndexCount(memgraph::storage::v3::LabelId label, memgraph::storage::PropertyId property, int64_t count) { +// for (auto &index : label_property_index_) { +// if (std::get<0>(index) == label && std::get<1>(index) == property) { +// std::get<2>(index) = count; +// return; +// } +// } +// label_property_index_.emplace_back(label, property, count); +// } + +// memgraph::storage::v3::LabelId NameToLabel(const std::string &name) { +// auto found = primary_labels_.find(name); +// if (found != primary_labels_.end()) return found->second; +// return primary_labels_.emplace(name, +// memgraph::storage::v3::LabelId::FromUint(primary_labels_.size())).first->second; +// } + +// memgraph::storage::v3::LabelId Label(const std::string &name) { return NameToLabel(name); } + +// memgraph::storage::EdgeTypeId NameToEdgeType(const std::string &name) { +// auto found = edge_types_.find(name); +// if (found != edge_types_.end()) return found->second; +// return edge_types_.emplace(name, memgraph::storage::EdgeTypeId::FromUint(edge_types_.size())).first->second; +// } + +// memgraph::storage::PropertyId NameToPrimaryProperty(const std::string &name) { +// auto found = primary_properties_.find(name); +// if (found != primary_properties_.end()) return found->second; +// return primary_properties_.emplace(name, +// memgraph::storage::PropertyId::FromUint(primary_properties_.size())).first->second; +// } + +// memgraph::storage::PropertyId NameToSecondaryProperty(const std::string &name) { +// auto found = secondary_properties_.find(name); +// if (found != secondary_properties_.end()) return found->second; +// return secondary_properties_.emplace(name, +// memgraph::storage::PropertyId::FromUint(secondary_properties_.size())).first->second; +// } + +// memgraph::storage::PropertyId PrimaryProperty(const std::string &name) { return NameToPrimaryProperty(name); } +// memgraph::storage::PropertyId SecondaryProperty(const std::string &name) { return NameToSecondaryProperty(name); } + +// std::string PrimaryPropertyToName(memgraph::storage::PropertyId property) const { +// for (const auto &kv : primary_properties_) { +// if (kv.second == property) return kv.first; +// } +// LOG_FATAL("Unable to find primary property name"); +// } + +// std::string SecondaryPropertyToName(memgraph::storage::PropertyId property) const { +// for (const auto &kv : secondary_properties_) { +// if (kv.second == property) return kv.first; +// } +// LOG_FATAL("Unable to find secondary property name"); +// } + +// std::string PrimaryPropertyName(memgraph::storage::PropertyId property) const { return +// PrimaryPropertyToName(property); } std::string SecondaryPropertyName(memgraph::storage::PropertyId property) const +// { return SecondaryPropertyToName(property); } + +// private: +// std::unordered_map primary_labels_; +// std::unordered_map secondary_labels_; +// std::unordered_map edge_types_; +// std::unordered_map primary_properties_; +// std::unordered_map secondary_properties_; + +// std::unordered_map label_index_; +// std::vector> +// label_property_index_; +// }; + } // namespace memgraph::query::plan diff --git a/tests/unit/query_plan_checker_v2.hpp b/tests/unit/query_plan_checker_v2.hpp new file mode 100644 index 000000000..6a6a12680 --- /dev/null +++ b/tests/unit/query_plan_checker_v2.hpp @@ -0,0 +1,559 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include + +#include "query/frontend/semantic/symbol_generator.hpp" +#include "query/frontend/semantic/symbol_table.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/plan/planner.hpp" +#include "query/v2/plan/preprocess.hpp" + +namespace memgraph::query::v2::plan { + +class BaseOpChecker { + public: + virtual ~BaseOpChecker() {} + + virtual void CheckOp(LogicalOperator &, const SymbolTable &) = 0; +}; + +class PlanChecker : public virtual HierarchicalLogicalOperatorVisitor { + public: + using HierarchicalLogicalOperatorVisitor::PostVisit; + using HierarchicalLogicalOperatorVisitor::PreVisit; + using HierarchicalLogicalOperatorVisitor::Visit; + + PlanChecker(const std::list> &checkers, const SymbolTable &symbol_table) + : symbol_table_(symbol_table) { + for (const auto &checker : checkers) checkers_.emplace_back(checker.get()); + } + + PlanChecker(const std::list &checkers, const SymbolTable &symbol_table) + : checkers_(checkers), symbol_table_(symbol_table) {} + +#define PRE_VISIT(TOp) \ + bool PreVisit(TOp &op) override { \ + CheckOp(op); \ + return true; \ + } + +#define VISIT(TOp) \ + bool Visit(TOp &op) override { \ + CheckOp(op); \ + return true; \ + } + + PRE_VISIT(CreateNode); + PRE_VISIT(CreateExpand); + PRE_VISIT(Delete); + PRE_VISIT(ScanAll); + PRE_VISIT(ScanAllByLabel); + PRE_VISIT(ScanAllByLabelPropertyValue); + PRE_VISIT(ScanAllByLabelPropertyRange); + PRE_VISIT(ScanAllByLabelProperty); + PRE_VISIT(Expand); + PRE_VISIT(ExpandVariable); + PRE_VISIT(Filter); + PRE_VISIT(ConstructNamedPath); + PRE_VISIT(Produce); + PRE_VISIT(SetProperty); + PRE_VISIT(SetProperties); + PRE_VISIT(SetLabels); + PRE_VISIT(RemoveProperty); + PRE_VISIT(RemoveLabels); + PRE_VISIT(EdgeUniquenessFilter); + PRE_VISIT(Accumulate); + PRE_VISIT(Aggregate); + PRE_VISIT(Skip); + PRE_VISIT(Limit); + PRE_VISIT(OrderBy); + bool PreVisit(Merge &op) override { + CheckOp(op); + op.input()->Accept(*this); + return false; + } + bool PreVisit(Optional &op) override { + CheckOp(op); + op.input()->Accept(*this); + return false; + } + PRE_VISIT(Unwind); + PRE_VISIT(Distinct); + + bool PreVisit(Foreach &op) override { + CheckOp(op); + return false; + } + + bool Visit(Once &) override { + // Ignore checking Once, it is implicitly at the end. + return true; + } + + bool PreVisit(Cartesian &op) override { + CheckOp(op); + return false; + } + + PRE_VISIT(CallProcedure); + +#undef PRE_VISIT +#undef VISIT + + void CheckOp(LogicalOperator &op) { + ASSERT_FALSE(checkers_.empty()); + checkers_.back()->CheckOp(op, symbol_table_); + checkers_.pop_back(); + } + + std::list checkers_; + const SymbolTable &symbol_table_; +}; + +template +class OpChecker : public BaseOpChecker { + public: + void CheckOp(LogicalOperator &op, const SymbolTable &symbol_table) override { + auto *expected_op = dynamic_cast(&op); + ASSERT_TRUE(expected_op) << "op is '" << op.GetTypeInfo().name << "' expected '" << TOp::kType.name << "'!"; + ExpectOp(*expected_op, symbol_table); + } + + virtual void ExpectOp(TOp &, const SymbolTable &) {} +}; + +// using ExpectScanAllByPrimaryKey = OpChecker; + +using ExpectCreateNode = OpChecker; +using ExpectCreateExpand = OpChecker; +using ExpectDelete = OpChecker; +using ExpectScanAll = OpChecker; +using ExpectScanAllByLabel = OpChecker; +// using ExpectScanAllById = OpChecker; +using ExpectExpand = OpChecker; +using ExpectFilter = OpChecker; +using ExpectConstructNamedPath = OpChecker; +using ExpectProduce = OpChecker; +using ExpectSetProperty = OpChecker; +using ExpectSetProperties = OpChecker; +using ExpectSetLabels = OpChecker; +using ExpectRemoveProperty = OpChecker; +using ExpectRemoveLabels = OpChecker; +using ExpectEdgeUniquenessFilter = OpChecker; +using ExpectSkip = OpChecker; +using ExpectLimit = OpChecker; +using ExpectOrderBy = OpChecker; +using ExpectUnwind = OpChecker; +using ExpectDistinct = OpChecker; + +// class ExpectForeach : public OpChecker { +// public: +// ExpectForeach(const std::list &input, const std::list &updates) +// : input_(input), updates_(updates) {} + +// void ExpectOp(Foreach &foreach, const SymbolTable &symbol_table) override { +// PlanChecker check_input(input_, symbol_table); +// foreach +// .input_->Accept(check_input); +// PlanChecker check_updates(updates_, symbol_table); +// foreach +// .update_clauses_->Accept(check_updates); +// } + +// private: +// std::list input_; +// std::list updates_; +// }; + +// class ExpectExpandVariable : public OpChecker { +// public: +// void ExpectOp(ExpandVariable &op, const SymbolTable &) override { +// EXPECT_EQ(op.type_, memgraph::query::EdgeAtom::Type::DEPTH_FIRST); +// } +// }; + +// class ExpectExpandBfs : public OpChecker { +// public: +// void ExpectOp(ExpandVariable &op, const SymbolTable &) override { +// EXPECT_EQ(op.type_, memgraph::query::EdgeAtom::Type::BREADTH_FIRST); +// } +// }; + +// class ExpectAccumulate : public OpChecker { +// public: +// explicit ExpectAccumulate(const std::unordered_set &symbols) : symbols_(symbols) {} + +// void ExpectOp(Accumulate &op, const SymbolTable &) override { +// std::unordered_set got_symbols(op.symbols_.begin(), op.symbols_.end()); +// EXPECT_EQ(symbols_, got_symbols); +// } + +// private: +// const std::unordered_set symbols_; +// }; + +// class ExpectAggregate : public OpChecker { +// public: +// ExpectAggregate(const std::vector &aggregations, +// const std::unordered_set &group_by) +// : aggregations_(aggregations), group_by_(group_by) {} + +// void ExpectOp(Aggregate &op, const SymbolTable &symbol_table) override { +// auto aggr_it = aggregations_.begin(); +// for (const auto &aggr_elem : op.aggregations_) { +// ASSERT_NE(aggr_it, aggregations_.end()); +// auto aggr = *aggr_it++; +// // TODO: Proper expression equality +// EXPECT_EQ(typeid(aggr_elem.value).hash_code(), typeid(aggr->expression1_).hash_code()); +// EXPECT_EQ(typeid(aggr_elem.key).hash_code(), typeid(aggr->expression2_).hash_code()); +// EXPECT_EQ(aggr_elem.op, aggr->op_); +// EXPECT_EQ(aggr_elem.output_sym, symbol_table.at(*aggr)); +// } +// EXPECT_EQ(aggr_it, aggregations_.end()); +// // TODO: Proper group by expression equality +// std::unordered_set got_group_by; +// for (auto *expr : op.group_by_) got_group_by.insert(typeid(*expr).hash_code()); +// std::unordered_set expected_group_by; +// for (auto *expr : group_by_) expected_group_by.insert(typeid(*expr).hash_code()); +// EXPECT_EQ(got_group_by, expected_group_by); +// } + +// private: +// std::vector aggregations_; +// std::unordered_set group_by_; +// }; + +// class ExpectMerge : public OpChecker { +// public: +// ExpectMerge(const std::list &on_match, const std::list &on_create) +// : on_match_(on_match), on_create_(on_create) {} + +// void ExpectOp(Merge &merge, const SymbolTable &symbol_table) override { +// PlanChecker check_match(on_match_, symbol_table); +// merge.merge_match_->Accept(check_match); +// PlanChecker check_create(on_create_, symbol_table); +// merge.merge_create_->Accept(check_create); +// } + +// private: +// const std::list &on_match_; +// const std::list &on_create_; +// }; + +// class ExpectOptional : public OpChecker { +// public: +// explicit ExpectOptional(const std::list &optional) : optional_(optional) {} + +// ExpectOptional(const std::vector &optional_symbols, const std::list &optional) +// : optional_symbols_(optional_symbols), optional_(optional) {} + +// void ExpectOp(Optional &optional, const SymbolTable &symbol_table) override { +// if (!optional_symbols_.empty()) { +// EXPECT_THAT(optional.optional_symbols_, testing::UnorderedElementsAreArray(optional_symbols_)); +// } +// PlanChecker check_optional(optional_, symbol_table); +// optional.optional_->Accept(check_optional); +// } + +// private: +// std::vector optional_symbols_; +// const std::list &optional_; +// }; + +// class ExpectScanAllByLabelPropertyValue : public OpChecker { +// public: +// ExpectScanAllByLabelPropertyValue(memgraph::storage::LabelId label, +// const std::pair &prop_pair, +// memgraph::query::Expression *expression) +// : label_(label), property_(prop_pair.second), expression_(expression) {} + +// void ExpectOp(ScanAllByLabelPropertyValue &scan_all, const SymbolTable &) override { +// EXPECT_EQ(scan_all.label_, label_); +// EXPECT_EQ(scan_all.property_, property_); +// // TODO: Proper expression equality +// EXPECT_EQ(typeid(scan_all.expression_).hash_code(), typeid(expression_).hash_code()); +// } + +// private: +// memgraph::storage::LabelId label_; +// memgraph::storage::PropertyId property_; +// memgraph::query::Expression *expression_; +// }; + +// class ExpectScanAllByLabelPropertyRange : public OpChecker { +// public: +// ExpectScanAllByLabelPropertyRange(memgraph::storage::LabelId label, memgraph::storage::PropertyId property, +// std::optional lower_bound, +// std::optional upper_bound) +// : label_(label), property_(property), lower_bound_(lower_bound), upper_bound_(upper_bound) {} + +// void ExpectOp(ScanAllByLabelPropertyRange &scan_all, const SymbolTable &) override { +// EXPECT_EQ(scan_all.label_, label_); +// EXPECT_EQ(scan_all.property_, property_); +// if (lower_bound_) { +// ASSERT_TRUE(scan_all.lower_bound_); +// // TODO: Proper expression equality +// EXPECT_EQ(typeid(scan_all.lower_bound_->value()).hash_code(), typeid(lower_bound_->value()).hash_code()); +// EXPECT_EQ(scan_all.lower_bound_->type(), lower_bound_->type()); +// } +// if (upper_bound_) { +// ASSERT_TRUE(scan_all.upper_bound_); +// // TODO: Proper expression equality +// EXPECT_EQ(typeid(scan_all.upper_bound_->value()).hash_code(), typeid(upper_bound_->value()).hash_code()); +// EXPECT_EQ(scan_all.upper_bound_->type(), upper_bound_->type()); +// } +// } + +// private: +// memgraph::storage::LabelId label_; +// memgraph::storage::PropertyId property_; +// std::optional lower_bound_; +// std::optional upper_bound_; +// }; + +// class ExpectScanAllByLabelProperty : public OpChecker { +// public: +// ExpectScanAllByLabelProperty(memgraph::storage::LabelId label, +// const std::pair &prop_pair) +// : label_(label), property_(prop_pair.second) {} + +// void ExpectOp(ScanAllByLabelProperty &scan_all, const SymbolTable &) override { +// EXPECT_EQ(scan_all.label_, label_); +// EXPECT_EQ(scan_all.property_, property_); +// } + +// private: +// memgraph::storage::LabelId label_; +// memgraph::storage::PropertyId property_; +// }; + +class ExpectScanAllByPrimaryKey : public OpChecker { + public: + ExpectScanAllByPrimaryKey(memgraph::storage::v3::LabelId label, const std::vector &properties) + : label_(label), properties_(properties) {} + + void ExpectOp(v2::plan::ScanAllByPrimaryKey &scan_all, const SymbolTable &) override { + EXPECT_EQ(scan_all.label_, label_); + // EXPECT_EQ(scan_all.property_, property_); + + // TODO(gvolfing) maybe assert the size of the 2 vectors. + // TODO(gvolfing) maybe use some std alg if Expression lets us. + + bool primary_property_match = true; + for (const auto &expected_prop : properties_) { + bool has_match = false; + for (const auto &prop : scan_all.primary_key_) { + if (typeid(prop).hash_code() == typeid(expected_prop).hash_code()) { + has_match = true; + } + } + if (!has_match) { + primary_property_match = false; + } + } + + EXPECT_TRUE(primary_property_match); + } + + private: + memgraph::storage::v3::LabelId label_; + std::vector properties_; +}; + +class ExpectCartesian : public OpChecker { + public: + ExpectCartesian(const std::list> &left, + const std::list> &right) + : left_(left), right_(right) {} + + void ExpectOp(Cartesian &op, const SymbolTable &symbol_table) override { + ASSERT_TRUE(op.left_op_); + PlanChecker left_checker(left_, symbol_table); + op.left_op_->Accept(left_checker); + ASSERT_TRUE(op.right_op_); + PlanChecker right_checker(right_, symbol_table); + op.right_op_->Accept(right_checker); + } + + private: + const std::list> &left_; + const std::list> &right_; +}; + +class ExpectCallProcedure : public OpChecker { + public: + ExpectCallProcedure(const std::string &name, const std::vector &args, + const std::vector &fields, const std::vector &result_syms) + : name_(name), args_(args), fields_(fields), result_syms_(result_syms) {} + + void ExpectOp(CallProcedure &op, const SymbolTable &symbol_table) override { + EXPECT_EQ(op.procedure_name_, name_); + EXPECT_EQ(op.arguments_.size(), args_.size()); + for (size_t i = 0; i < args_.size(); ++i) { + const auto *op_arg = op.arguments_[i]; + const auto *expected_arg = args_[i]; + // TODO: Proper expression equality + EXPECT_EQ(op_arg->GetTypeInfo(), expected_arg->GetTypeInfo()); + } + EXPECT_EQ(op.result_fields_, fields_); + EXPECT_EQ(op.result_symbols_, result_syms_); + } + + private: + std::string name_; + std::vector args_; + std::vector fields_; + std::vector result_syms_; +}; + +template +std::list> MakeCheckers(T arg) { + std::list> l; + l.emplace_back(std::make_unique(arg)); + return l; +} + +template +std::list> MakeCheckers(T arg, Rest &&...rest) { + auto l = MakeCheckers(std::forward(rest)...); + l.emplace_front(std::make_unique(arg)); + return std::move(l); +} + +template +TPlanner MakePlanner(TDbAccessor *dba, AstStorage &storage, SymbolTable &symbol_table, CypherQuery *query) { + auto planning_context = MakePlanningContext(&storage, &symbol_table, query, dba); + auto query_parts = CollectQueryParts(symbol_table, storage, query); + auto single_query_parts = query_parts.query_parts.at(0).single_query_parts; + return TPlanner(single_query_parts, planning_context); +} + +class FakeDistributedDbAccessor { + public: + int64_t VerticesCount(memgraph::storage::v3::LabelId label) const { + auto found = label_index_.find(label); + if (found != label_index_.end()) return found->second; + return 0; + } + + int64_t VerticesCount(memgraph::storage::v3::LabelId label, memgraph::storage::v3::PropertyId property) const { + for (auto &index : label_property_index_) { + if (std::get<0>(index) == label && std::get<1>(index) == property) { + return std::get<2>(index); + } + } + return 0; + } + + bool LabelIndexExists(memgraph::storage::v3::LabelId label) const { + return label_index_.find(label) != label_index_.end(); + } + + bool LabelPropertyIndexExists(memgraph::storage::v3::LabelId label, + memgraph::storage::v3::PropertyId property) const { + for (auto &index : label_property_index_) { + if (std::get<0>(index) == label && std::get<1>(index) == property) { + return true; + } + } + return false; + } + + void SetIndexCount(memgraph::storage::v3::LabelId label, int64_t count) { label_index_[label] = count; } + + void SetIndexCount(memgraph::storage::v3::LabelId label, memgraph::storage::v3::PropertyId property, int64_t count) { + for (auto &index : label_property_index_) { + if (std::get<0>(index) == label && std::get<1>(index) == property) { + std::get<2>(index) = count; + return; + } + } + label_property_index_.emplace_back(label, property, count); + } + + memgraph::storage::v3::LabelId NameToLabel(const std::string &name) { + auto found = primary_labels_.find(name); + if (found != primary_labels_.end()) return found->second; + return primary_labels_.emplace(name, memgraph::storage::v3::LabelId::FromUint(primary_labels_.size())) + .first->second; + } + + memgraph::storage::v3::LabelId Label(const std::string &name) { return NameToLabel(name); } + + memgraph::storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) { + auto found = edge_types_.find(name); + if (found != edge_types_.end()) return found->second; + return edge_types_.emplace(name, memgraph::storage::v3::EdgeTypeId::FromUint(edge_types_.size())).first->second; + } + + memgraph::storage::v3::PropertyId NameToPrimaryProperty(const std::string &name) { + auto found = primary_properties_.find(name); + if (found != primary_properties_.end()) return found->second; + return primary_properties_.emplace(name, memgraph::storage::v3::PropertyId::FromUint(primary_properties_.size())) + .first->second; + } + + memgraph::storage::v3::PropertyId NameToSecondaryProperty(const std::string &name) { + auto found = secondary_properties_.find(name); + if (found != secondary_properties_.end()) return found->second; + return secondary_properties_ + .emplace(name, memgraph::storage::v3::PropertyId::FromUint(secondary_properties_.size())) + .first->second; + } + + memgraph::storage::v3::PropertyId PrimaryProperty(const std::string &name) { return NameToPrimaryProperty(name); } + memgraph::storage::v3::PropertyId SecondaryProperty(const std::string &name) { return NameToSecondaryProperty(name); } + + std::string PrimaryPropertyToName(memgraph::storage::v3::PropertyId property) const { + for (const auto &kv : primary_properties_) { + if (kv.second == property) return kv.first; + } + LOG_FATAL("Unable to find primary property name"); + } + + std::string SecondaryPropertyToName(memgraph::storage::v3::PropertyId property) const { + for (const auto &kv : secondary_properties_) { + if (kv.second == property) return kv.first; + } + LOG_FATAL("Unable to find secondary property name"); + } + + std::string PrimaryPropertyName(memgraph::storage::v3::PropertyId property) const { + return PrimaryPropertyToName(property); + } + std::string SecondaryPropertyName(memgraph::storage::v3::PropertyId property) const { + return SecondaryPropertyToName(property); + } + + memgraph::storage::v3::PropertyId NameToProperty(const std::string &name) { + return storage::v3::PropertyId::FromUint(0); + } + + std::vector ExtractPrimaryKey(storage::v3::LabelId label, + std::vector property_filters) { + return std::vector{}; + } + + private: + std::unordered_map primary_labels_; + std::unordered_map secondary_labels_; + std::unordered_map edge_types_; + std::unordered_map primary_properties_; + std::unordered_map secondary_properties_; + + std::unordered_map label_index_; + std::vector> + label_property_index_; +}; + +} // namespace memgraph::query::v2::plan diff --git a/tests/unit/query_plan_v2.cpp b/tests/unit/query_plan_v2.cpp new file mode 100644 index 000000000..009d0ca56 --- /dev/null +++ b/tests/unit/query_plan_v2.cpp @@ -0,0 +1,1664 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query_plan_checker_v2.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "query/v2/frontend/ast/ast.hpp" +// #include "query/frontend/semantic/symbol_generator.hpp" +#include "expr/semantic/symbol_generator.hpp" +#include "query/frontend/semantic/symbol_table.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/plan/planner.hpp" + +#include "query_common.hpp" + +namespace memgraph::query { +::std::ostream &operator<<(::std::ostream &os, const Symbol &sym) { + return os << "Symbol{\"" << sym.name() << "\" [" << sym.position() << "] " << Symbol::TypeToString(sym.type()) << "}"; +} +} // namespace memgraph::query + +// using namespace memgraph::query::v2::plan; +using namespace memgraph::expr::plan; +using memgraph::query::AstStorage; +using memgraph::query::SingleQuery; +using memgraph::query::Symbol; +using memgraph::query::SymbolGenerator; +using memgraph::query::v2::SymbolTable; +using Type = memgraph::query::v2::EdgeAtom::Type; +using Direction = memgraph::query::v2::EdgeAtom::Direction; +using Bound = ScanAllByLabelPropertyRange::Bound; + +namespace { + +class Planner { + public: + template + Planner(std::vector single_query_parts, PlanningContext context) { + memgraph::expr::Parameters parameters; + PostProcessor post_processor(parameters); + plan_ = MakeLogicalPlanForSingleQuery(single_query_parts, &context); + plan_ = post_processor.Rewrite(std::move(plan_), &context); + } + + auto &plan() { return *plan_; } + + private: + std::unique_ptr plan_; +}; + +template +auto CheckPlan(LogicalOperator &plan, const SymbolTable &symbol_table, TChecker... checker) { + std::list checkers{&checker...}; + PlanChecker plan_checker(checkers, symbol_table); + plan.Accept(plan_checker); + EXPECT_TRUE(plan_checker.checkers_.empty()); +} + +template +auto CheckPlan(memgraph::query::CypherQuery *query, AstStorage &storage, TChecker... checker) { + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, checker...); +} + +template +class TestPlanner : public ::testing::Test {}; + +using PlannerTypes = ::testing::Types; + +void DeleteListContent(std::list *list) { + for (BaseOpChecker *ptr : *list) { + delete ptr; + } +} +TYPED_TEST_CASE(TestPlanner, PlannerTypes); + +TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { + FakeDistributedDbAccessor dba; + auto label = dba.Label("prim_label_one"); + auto prim_prop_one = PRIMARY_PROPERTY_PAIR("prim_prop_one"); + // auto prim_prop_two = PRIMARY_PROPERTY_PAIR("prim_prop_two"); + auto sec_prop_one = PRIMARY_PROPERTY_PAIR("sec_prop_one"); + auto sec_prop_two = PRIMARY_PROPERTY_PAIR("sec_prop_two"); + auto sec_prop_three = PRIMARY_PROPERTY_PAIR("sec_prop_three"); + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, prim_prop_one.second, 1); + // dba.SetIndexCount(label, prim_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_one.second, 1); + dba.SetIndexCount(label, sec_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_three.second, 1); + memgraph::query::v2::AstStorage storage; + { + memgraph::query::v2::Expression *expected_primary_key; + + // Pray and hope for the best... + expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); + + // auto asd1 = NODE("n", "label"); + // auto asd2 = PATTERN(NODE("n", "label")); + // auto asd3 = MATCH_V2(PATTERN(NODE("n", "label"))); + // auto asd4 = WHERE_V2(PROPERTY_LOOKUP("n", prim_prop_one)); + + auto *query = QUERY(SINGLE_QUERY_V2(MATCH_V2(PATTERN(NODE("n", "label"))), + WHERE_V2(PROPERTY_LOOKUP("n", prim_prop_one)), RETURN("n"))); + auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByPrimaryKey(label, {expected_primary_key}), ExpectProduce()); + + // // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL RETURN n + // auto *query2 = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), + // WHERE(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop)))), RETURN("n"))); + // auto symbol_table = memgraph::query::MakeSymbolTable(query); + // auto planner = MakePlanner(&dba, storage, symbol_table, query); + // // We expect ScanAllByLabelProperty to come instead of ScanAll > Filter. + // CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectExpand(), + // ExpectProduce()); + } +} + +/* + + + + + + +TYPED_TEST(TestPlanner, MatchNodeReturn) { + // Test MATCH (n) RETURN n + AstStorage storage; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(as_n))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CreateNodeReturn) { + // Test CREATE (n) RETURN n AS n + AstStorage storage; + auto ident_n = IDENT("n"); + auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), RETURN(ident_n, AS("n")))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CreateExpand) { + // Test CREATE (n) -[r :rel1]-> (m) + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "relationship"; + auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); + CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand()); +} + +TYPED_TEST(TestPlanner, CreateMultipleNode) { + // Test CREATE (n), (m) + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n")), PATTERN(NODE("m"))))); + CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateNode()); +} + +TYPED_TEST(TestPlanner, CreateNodeExpandNode) { + // Test CREATE (n) -[r :rel]-> (m), (l) + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "rel"; + auto *query = QUERY(SINGLE_QUERY( + CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m")), PATTERN(NODE("l"))))); + CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectCreateNode()); +} + +TYPED_TEST(TestPlanner, CreateNamedPattern) { + // Test CREATE p = (n) -[r :rel]-> (m) + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "rel"; + auto *query = + QUERY(SINGLE_QUERY(CREATE(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); + CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectConstructNamedPath()); +} + +TYPED_TEST(TestPlanner, MatchCreateExpand) { + // Test MATCH (n) CREATE (n) -[r :rel1]-> (m) + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "relationship"; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), + CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); + CheckPlan(query, storage, ExpectScanAll(), ExpectCreateExpand()); +} + +TYPED_TEST(TestPlanner, MatchLabeledNodes) { + // Test MATCH (n :label) RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = "label"; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", label))), RETURN(as_n))); + { + // Without created label index + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); + } + { + // With created label index + dba.SetIndexCount(dba.Label(label), 0); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectProduce()); + } +} + +TYPED_TEST(TestPlanner, MatchPathReturn) { + // Test MATCH (n) -[r :relationship]- (m) RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "relationship"; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), RETURN(as_n))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchNamedPatternReturn) { + // Test MATCH p = (n) -[r :relationship]- (m) RETURN p + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "relationship"; + auto *as_p = NEXPR("p", IDENT("p")); + auto *query = QUERY(SINGLE_QUERY( + MATCH(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), RETURN(as_p))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectConstructNamedPath(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchNamedPatternWithPredicateReturn) { + // Test MATCH p = (n) -[r :relationship]- (m) WHERE 2 = p RETURN p + AstStorage storage; + FakeDistributedDbAccessor dba; + auto relationship = "relationship"; + auto *as_p = NEXPR("p", IDENT("p")); + auto *query = + QUERY(SINGLE_QUERY(MATCH(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), + WHERE(EQ(LITERAL(2), IDENT("p"))), RETURN(as_p))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectConstructNamedPath(), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) { + // Test OPTIONAL MATCH p = (n) -[r]- (m) RETURN p + AstStorage storage; + auto node_n = NODE("n"); + auto edge = EDGE("r"); + auto node_m = NODE("m"); + auto pattern = NAMED_PATTERN("p", node_n, edge, node_m); + auto as_p = AS("p"); + auto *query = QUERY(SINGLE_QUERY(OPTIONAL_MATCH(pattern), RETURN("p", as_p))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto get_symbol = [&symbol_table](const auto *ast_node) { return symbol_table.at(*ast_node->identifier_); }; + std::vector optional_symbols{get_symbol(pattern), get_symbol(node_n), get_symbol(edge), get_symbol(node_m)}; + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + std::list optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()}; + CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce()); + DeleteListContent(&optional); +} + +TYPED_TEST(TestPlanner, MatchWhereReturn) { + // Test MATCH (n) WHERE n.property < 42 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto property = dba.Property("property"); + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WHERE(LESS(PROPERTY_LOOKUP("n", property), LITERAL(42))), RETURN(as_n))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchDelete) { + // Test MATCH (n) DELETE n + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), DELETE(IDENT("n")))); + CheckPlan(query, storage, ExpectScanAll(), ExpectDelete()); +} + +TYPED_TEST(TestPlanner, MatchNodeSet) { + // Test MATCH (n) SET n.prop = 42, n = n, n :label + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + auto label = "label"; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), SET(PROPERTY_LOOKUP("n", prop), LITERAL(42)), + SET("n", IDENT("n")), SET("n", {label}))); + CheckPlan(query, storage, ExpectScanAll(), ExpectSetProperty(), ExpectSetProperties(), ExpectSetLabels()); +} + +TYPED_TEST(TestPlanner, MatchRemove) { + // Test MATCH (n) REMOVE n.prop REMOVE n :label + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + auto label = "label"; + auto *query = + QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), REMOVE(PROPERTY_LOOKUP("n", prop)), REMOVE("n", {label}))); + CheckPlan(query, storage, ExpectScanAll(), ExpectRemoveProperty(), ExpectRemoveLabels()); +} + +TYPED_TEST(TestPlanner, MatchMultiPattern) { + // Test MATCH (n) -[r]- (m), (j) -[e]- (i) RETURN n + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY( + MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m")), PATTERN(NODE("j"), EDGE("e"), NODE("i"))), RETURN("n"))); + // We expect the expansions after the first to have a uniqueness filter in a + // single MATCH clause. + CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectScanAll(), ExpectExpand(), + ExpectEdgeUniquenessFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchMultiPatternSameStart) { + // Test MATCH (n), (n) -[e]- (m) RETURN n + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n")), PATTERN(NODE("n"), EDGE("e"), NODE("m"))), RETURN("n"))); + // We expect the second pattern to generate only an Expand, since another + // ScanAll would be redundant. + CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchMultiPatternSameExpandStart) { + // Test MATCH (n) -[r]- (m), (m) -[e]- (l) RETURN n + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY( + MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m")), PATTERN(NODE("m"), EDGE("e"), NODE("l"))), RETURN("n"))); + // We expect the second pattern to generate only an Expand. Another + // ScanAll would be redundant, as it would generate the nodes obtained from + // expansion. Additionally, a uniqueness filter is expected. + CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectExpand(), ExpectEdgeUniquenessFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MultiMatch) { + // Test MATCH (n) -[r]- (m) MATCH (j) -[e]- (i) -[f]- (h) RETURN n + AstStorage storage; + auto *node_n = NODE("n"); + auto *edge_r = EDGE("r"); + auto *node_m = NODE("m"); + auto *node_j = NODE("j"); + auto *edge_e = EDGE("e"); + auto *node_i = NODE("i"); + auto *edge_f = EDGE("f"); + auto *node_h = NODE("h"); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n, edge_r, node_m)), + MATCH(PATTERN(node_j, edge_e, node_i, edge_f, node_h)), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // Multiple MATCH clauses form a Cartesian product, so the uniqueness should + // not cross MATCH boundaries. + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectScanAll(), ExpectExpand(), + ExpectExpand(), ExpectEdgeUniquenessFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MultiMatchSameStart) { + // Test MATCH (n) MATCH (n) -[r]- (m) RETURN n + AstStorage storage; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = + QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), RETURN(as_n))); + // Similar to MatchMultiPatternSameStart, we expect only Expand from second + // MATCH clause. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchWithReturn) { + // Test MATCH (old) WITH old AS new RETURN new + AstStorage storage; + auto *as_new = NEXPR("new", IDENT("new")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("old"))), WITH("old", AS("new")), RETURN(as_new))); + // No accumulation since we only do reads. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchWithWhereReturn) { + // Test MATCH (old) WITH old AS new WHERE new.prop < 42 RETURN new + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *as_new = NEXPR("new", IDENT("new")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("old"))), WITH("old", AS("new")), + WHERE(LESS(PROPERTY_LOOKUP("new", prop), LITERAL(42))), RETURN(as_new))); + // No accumulation since we only do reads. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CreateMultiExpand) { + // Test CREATE (n) -[r :r]-> (m), (n) - [p :p]-> (l) + FakeDistributedDbAccessor dba; + auto r = "r"; + auto p = "p"; + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {r}), NODE("m")), + PATTERN(NODE("n"), EDGE("p", Direction::OUT, {p}), NODE("l"))))); + CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectCreateExpand()); +} + +TYPED_TEST(TestPlanner, MatchWithSumWhereReturn) { + // Test MATCH (n) WITH SUM(n.prop) + 42 AS sum WHERE sum < 42 + // RETURN sum AS result + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto sum = SUM(PROPERTY_LOOKUP("n", prop)); + auto literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WITH(ADD(sum, literal), AS("sum")), + WHERE(LESS(IDENT("sum"), LITERAL(42))), RETURN("sum", AS("result")))); + auto aggr = ExpectAggregate({sum}, {literal}); + CheckPlan(query, storage, ExpectScanAll(), aggr, ExpectProduce(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchReturnSum) { + // Test MATCH (n) RETURN SUM(n.prop1) AS sum, n.prop2 AS group + FakeDistributedDbAccessor dba; + auto prop1 = dba.Property("prop1"); + auto prop2 = dba.Property("prop2"); + AstStorage storage; + auto sum = SUM(PROPERTY_LOOKUP("n", prop1)); + auto n_prop2 = PROPERTY_LOOKUP("n", prop2); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(sum, AS("sum"), n_prop2, AS("group")))); + auto aggr = ExpectAggregate({sum}, {n_prop2}); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CreateWithSum) { + // Test CREATE (n) WITH SUM(n.prop) AS sum + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto ident_n = IDENT("n"); + auto n_prop = PROPERTY_LOOKUP(ident_n, prop); + auto sum = SUM(n_prop); + auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), WITH(sum, AS("sum")))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); + auto aggr = ExpectAggregate({sum}, {}); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect both the accumulation and aggregation because the part before + // WITH updates the database. + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchWithCreate) { + // Test MATCH (n) WITH n AS a CREATE (a) -[r :r]-> (b) + FakeDistributedDbAccessor dba; + auto r_type = "r"; + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WITH("n", AS("a")), + CREATE(PATTERN(NODE("a"), EDGE("r", Direction::OUT, {r_type}), NODE("b"))))); + CheckPlan(query, storage, ExpectScanAll(), ExpectProduce(), ExpectCreateExpand()); +} + +TYPED_TEST(TestPlanner, MatchReturnSkipLimit) { + // Test MATCH (n) RETURN n SKIP 2 LIMIT 1 + AstStorage storage; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(as_n, SKIP(LITERAL(2)), LIMIT(LITERAL(1))))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectSkip(), ExpectLimit()); +} + +TYPED_TEST(TestPlanner, CreateWithSkipReturnLimit) { + // Test CREATE (n) WITH n AS m SKIP 2 RETURN m LIMIT 1 + AstStorage storage; + auto ident_n = IDENT("n"); + auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), WITH(ident_n, AS("m"), SKIP(LITERAL(2))), + RETURN("m", LIMIT(LITERAL(1))))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // Since we have a write query, we need to have Accumulate. This is a bit + // different than Neo4j 3.0, which optimizes WITH followed by RETURN as a + // single RETURN clause and then moves Skip and Limit before Accumulate. + // This causes different behaviour. A newer version of Neo4j does the same + // thing as us here (but who knows if they change it again). + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, ExpectProduce(), ExpectSkip(), ExpectProduce(), + ExpectLimit()); +} + +TYPED_TEST(TestPlanner, CreateReturnSumSkipLimit) { + // Test CREATE (n) RETURN SUM(n.prop) AS s SKIP 2 LIMIT 1 + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto ident_n = IDENT("n"); + auto n_prop = PROPERTY_LOOKUP(ident_n, prop); + auto sum = SUM(n_prop); + auto query = + QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), RETURN(sum, AS("s"), SKIP(LITERAL(2)), LIMIT(LITERAL(1))))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); + auto aggr = ExpectAggregate({sum}, {}); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce(), ExpectSkip(), ExpectLimit()); +} + +TYPED_TEST(TestPlanner, MatchReturnOrderBy) { + // Test MATCH (n) RETURN n AS m ORDER BY n.prop + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *as_m = NEXPR("m", IDENT("n")); + auto *node_n = NODE("n"); + auto ret = RETURN(as_m, ORDER_BY(PROPERTY_LOOKUP("n", prop))); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n)), ret)); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectOrderBy()); +} + +TYPED_TEST(TestPlanner, CreateWithOrderByWhere) { + // Test CREATE (n) -[r :r]-> (m) + // WITH n AS new ORDER BY new.prop, r.prop WHERE m.prop < 42 + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + auto r_type = "r"; + AstStorage storage; + auto ident_n = IDENT("n"); + auto ident_r = IDENT("r"); + auto ident_m = IDENT("m"); + auto new_prop = PROPERTY_LOOKUP("new", prop); + auto r_prop = PROPERTY_LOOKUP(ident_r, prop); + auto m_prop = PROPERTY_LOOKUP(ident_m, prop); + auto query = + QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {r_type}), NODE("m"))), + WITH(ident_n, AS("new"), ORDER_BY(new_prop, r_prop)), WHERE(LESS(m_prop, LITERAL(42))))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + // Since this is a write query, we expect to accumulate to old used symbols. + auto acc = ExpectAccumulate({ + symbol_table.at(*ident_n), // `n` in WITH + symbol_table.at(*ident_r), // `r` in ORDER BY + symbol_table.at(*ident_m), // `m` in WHERE + }); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), ExpectCreateExpand(), acc, ExpectProduce(), + ExpectOrderBy(), ExpectFilter()); +} + +TYPED_TEST(TestPlanner, ReturnAddSumCountOrderBy) { + // Test RETURN SUM(1) + COUNT(2) AS result ORDER BY result + AstStorage storage; + auto sum = SUM(LITERAL(1)); + auto count = COUNT(LITERAL(2)); + auto *query = QUERY(SINGLE_QUERY(RETURN(ADD(sum, count), AS("result"), ORDER_BY(IDENT("result"))))); + auto aggr = ExpectAggregate({sum, count}, {}); + CheckPlan(query, storage, aggr, ExpectProduce(), ExpectOrderBy()); +} + +TYPED_TEST(TestPlanner, MatchMerge) { + // Test MATCH (n) MERGE (n) -[r :r]- (m) + // ON MATCH SET n.prop = 42 ON CREATE SET m = n + // RETURN n AS n + FakeDistributedDbAccessor dba; + auto r_type = "r"; + auto prop = dba.Property("prop"); + AstStorage storage; + auto ident_n = IDENT("n"); + auto query = + QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), + MERGE(PATTERN(NODE("n"), EDGE("r", Direction::BOTH, {r_type}), NODE("m")), + ON_MATCH(SET(PROPERTY_LOOKUP("n", prop), LITERAL(42))), ON_CREATE(SET("m", IDENT("n")))), + RETURN(ident_n, AS("n")))); + std::list on_match{new ExpectExpand(), new ExpectSetProperty()}; + std::list on_create{new ExpectCreateExpand(), new ExpectSetProperties()}; + auto symbol_table = memgraph::query::MakeSymbolTable(query); + // We expect Accumulate after Merge, because it is considered as a write. + auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce()); + DeleteListContent(&on_match); + DeleteListContent(&on_create); +} + +TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) { + // Test MATCH (n) OPTIONAL MATCH (n) -[r]- (m) WHERE m.prop < 42 RETURN r + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), + WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r"))); + std::list optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()}; + CheckPlan(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce()); + DeleteListContent(&optional); +} + +TYPED_TEST(TestPlanner, MatchUnwindReturn) { + // Test MATCH (n) UNWIND [1,2,3] AS x RETURN n, x + AstStorage storage; + auto *as_n = NEXPR("n", IDENT("n")); + auto *as_x = NEXPR("x", IDENT("x")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), UNWIND(LIST(LITERAL(1), LITERAL(2), LITERAL(3)), AS("x")), + RETURN(as_n, as_x))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectUnwind(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ReturnDistinctOrderBySkipLimit) { + // Test RETURN DISTINCT 1 ORDER BY 1 SKIP 1 LIMIT 1 + AstStorage storage; + auto *query = QUERY( + SINGLE_QUERY(RETURN_DISTINCT(LITERAL(1), AS("1"), ORDER_BY(LITERAL(1)), SKIP(LITERAL(1)), LIMIT(LITERAL(1))))); + CheckPlan(query, storage, ExpectProduce(), ExpectDistinct(), ExpectOrderBy(), ExpectSkip(), ExpectLimit()); +} + +TYPED_TEST(TestPlanner, CreateWithDistinctSumWhereReturn) { + // Test CREATE (n) WITH DISTINCT SUM(n.prop) AS s WHERE s < 42 RETURN s + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto node_n = NODE("n"); + auto sum = SUM(PROPERTY_LOOKUP("n", prop)); + auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(node_n)), WITH_DISTINCT(sum, AS("s")), + WHERE(LESS(IDENT("s"), LITERAL(42))), RETURN("s"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto acc = ExpectAccumulate({symbol_table.at(*node_n->identifier_)}); + auto aggr = ExpectAggregate({sum}, {}); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce(), ExpectDistinct(), + ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchCrossReferenceVariable) { + // Test MATCH (n {prop: m.prop}), (m {prop: n.prop}) RETURN n + FakeDistributedDbAccessor dba; + auto prop = PROPERTY_PAIR("prop"); + AstStorage storage; + auto node_n = NODE("n"); + auto m_prop = PROPERTY_LOOKUP("m", prop.second); + std::get<0>(node_n->properties_)[storage.GetPropertyIx(prop.first)] = m_prop; + auto node_m = NODE("m"); + auto n_prop = PROPERTY_LOOKUP("n", prop.second); + std::get<0>(node_m->properties_)[storage.GetPropertyIx(prop.first)] = n_prop; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n), PATTERN(node_m)), RETURN("n"))); + // We expect both ScanAll to come before filters (2 are joined into one), + // because they need to populate the symbol values. + CheckPlan(query, storage, ExpectScanAll(), ExpectScanAll(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchWhereBeforeExpand) { + // Test MATCH (n) -[r]- (m) WHERE n.prop < 42 RETURN n + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *as_n = NEXPR("n", IDENT("n")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), + WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN(as_n))); + // We expect Filter to come immediately after ScanAll, since it only uses `n`. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto prop = PROPERTY_PAIR("prop"); + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, prop.second, 1); + AstStorage storage; + + { + // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL RETURN n + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), + WHERE(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop)))), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect ScanAllByLabelProperty to come instead of ScanAll > Filter. + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectExpand(), ExpectProduce()); + } + + { + // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL OR true RETURN n + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), + WHERE(OR(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop))), LITERAL(true))), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect ScanAllBy > Filter because of the "or true" condition. + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectProduce()); + } + + { + // Test MATCH (n :label) -[r]- (m) + // WHERE n.prop IS NOT NULL AND n.x = 2 RETURN n + auto prop_x = PROPERTY_PAIR("x"); + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), + WHERE(AND(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop))), EQ(PROPERTY_LOOKUP("n", prop_x), LITERAL(2)))), + RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect ScanAllByLabelProperty > Filter + // to come instead of ScanAll > Filter. + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectFilter(), ExpectExpand(), + ExpectProduce()); + } +} + +TYPED_TEST(TestPlanner, MultiMatchWhere) { + // Test MATCH (n) -[r]- (m) MATCH (l) WHERE n.prop < 42 RETURN n + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), MATCH(PATTERN(NODE("l"))), + WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN("n"))); + // Even though WHERE is in the second MATCH clause, we expect Filter to come + // before second ScanAll, since it only uses the value from first ScanAll. + CheckPlan(query, storage, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectScanAll(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) { + // Test MATCH (n) -[r]- (m) OPTIONAL MATCH (l) WHERE n.prop < 42 RETURN n + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), OPTIONAL_MATCH(PATTERN(NODE("l"))), + WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN("n"))); + // Even though WHERE is in the second MATCH clause, and it uses the value from + // first ScanAll, it must remain part of the Optional. It should come before + // optional ScanAll. + std::list optional{new ExpectFilter(), new ExpectScanAll()}; + CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce()); + DeleteListContent(&optional); +} + +TYPED_TEST(TestPlanner, MatchReturnAsterisk) { + // Test MATCH (n) -[e]- (m) RETURN *, m.prop + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto ret = RETURN(PROPERTY_LOOKUP("m", prop), AS("m.prop")); + ret->body_.all_identifiers = true; + auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("e"), NODE("m"))), ret)); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); + std::vector output_names; + for (const auto &output_symbol : planner.plan().OutputSymbols(symbol_table)) { + output_names.emplace_back(output_symbol.name()); + } + std::vector expected_names{"e", "m", "n", "m.prop"}; + EXPECT_EQ(output_names, expected_names); +} + +TYPED_TEST(TestPlanner, MatchReturnAsteriskSum) { + // Test MATCH (n) RETURN *, SUM(n.prop) AS s + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + AstStorage storage; + auto sum = SUM(PROPERTY_LOOKUP("n", prop)); + auto ret = RETURN(sum, AS("s")); + ret->body_.all_identifiers = true; + auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), ret)); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + auto *produce = dynamic_cast(&planner.plan()); + ASSERT_TRUE(produce); + const auto &named_expressions = produce->named_expressions_; + ASSERT_EQ(named_expressions.size(), 2); + auto *expanded_ident = dynamic_cast(named_expressions[0]->expression_); + ASSERT_TRUE(expanded_ident); + auto aggr = ExpectAggregate({sum}, {expanded_ident}); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), aggr, ExpectProduce()); + std::vector output_names; + for (const auto &output_symbol : planner.plan().OutputSymbols(symbol_table)) { + output_names.emplace_back(output_symbol.name()); + } + std::vector expected_names{"n", "s"}; + EXPECT_EQ(output_names, expected_names); +} + +TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) { + // Test UNWIND [1] AS i MERGE (n {prop: i}) + AstStorage storage; + FakeDistributedDbAccessor dba; + auto node_n = NODE("n"); + std::get<0>(node_n->properties_)[storage.GetPropertyIx("prop")] = IDENT("i"); + auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1)), AS("i")), MERGE(PATTERN(node_n)))); + std::list on_match{new ExpectScanAll(), new ExpectFilter()}; + std::list on_create{new ExpectCreateNode()}; + CheckPlan(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create)); + DeleteListContent(&on_match); + DeleteListContent(&on_create); +} + +TYPED_TEST(TestPlanner, UnwindMergeNodePropertyWithIndex) { + // Test UNWIND [1] AS i MERGE (n :label {prop: i}) with label-property index + AstStorage storage; + FakeDistributedDbAccessor dba; + const auto label_name = "label"; + const auto label = dba.Label(label_name); + const auto property = PROPERTY_PAIR("prop"); + dba.SetIndexCount(label, property.second, 1); + auto node_n = NODE("n", label_name); + std::get<0>(node_n->properties_)[storage.GetPropertyIx(property.first)] = IDENT("i"); + auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1)), AS("i")), MERGE(PATTERN(node_n)))); + std::list on_match{new ExpectScanAllByLabelPropertyValue(label, property, IDENT("i"))}; + std::list on_create{new ExpectCreateNode()}; + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectUnwind(), ExpectMerge(on_match, on_create)); + DeleteListContent(&on_match); + DeleteListContent(&on_create); +} + +TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) { + // Test OPTIONAL MATCH (n) OPTIONAL MATCH (m) RETURN n + AstStorage storage; + auto *query = + QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n"))); + std::list optional{new ExpectScanAll()}; + CheckPlan(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce()); + DeleteListContent(&optional); +} + +TYPED_TEST(TestPlanner, FunctionAggregationReturn) { + // Test RETURN sqrt(SUM(2)) AS result, 42 AS group_by + AstStorage storage; + auto sum = SUM(LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN(FN("sqrt", sum), AS("result"), group_by_literal, AS("group_by")))); + auto aggr = ExpectAggregate({sum}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, FunctionWithoutArguments) { + // Test RETURN pi() AS pi + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(RETURN(FN("pi"), AS("pi")))); + CheckPlan(query, storage, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ListLiteralAggregationReturn) { + // Test RETURN [SUM(2)] AS result, 42 AS group_by + AstStorage storage; + auto sum = SUM(LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN(LIST(sum), AS("result"), group_by_literal, AS("group_by")))); + auto aggr = ExpectAggregate({sum}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MapLiteralAggregationReturn) { + // Test RETURN {sum: SUM(2)} AS result, 42 AS group_by + AstStorage storage; + FakeDistributedDbAccessor dba; + auto sum = SUM(LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY( + SINGLE_QUERY(RETURN(MAP({storage.GetPropertyIx("sum"), sum}), AS("result"), group_by_literal, AS("group_by")))); + auto aggr = ExpectAggregate({sum}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, EmptyListIndexAggregation) { + // Test RETURN [][SUM(2)] AS result, 42 AS group_by + AstStorage storage; + auto sum = SUM(LITERAL(2)); + auto empty_list = LIST(); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN(storage.Create(empty_list, sum), + AS("result"), group_by_literal, AS("group_by")))); + // We expect to group by '42' and the empty list, because it is a + // sub-expression of a binary operator which contains an aggregation. This is + // similar to grouping by '1' in `RETURN 1 + SUM(2)`. + auto aggr = ExpectAggregate({sum}, {empty_list, group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ListSliceAggregationReturn) { + // Test RETURN [1, 2][0..SUM(2)] AS result, 42 AS group_by + AstStorage storage; + auto sum = SUM(LITERAL(2)); + auto list = LIST(LITERAL(1), LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = + QUERY(SINGLE_QUERY(RETURN(SLICE(list, LITERAL(0), sum), AS("result"), group_by_literal, AS("group_by")))); + // Similarly to EmptyListIndexAggregation test, we expect grouping by list and + // '42', because slicing is an operator. + auto aggr = ExpectAggregate({sum}, {list, group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ListWithAggregationAndGroupBy) { + // Test RETURN [sum(2), 42] + AstStorage storage; + auto sum = SUM(LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN(LIST(sum, group_by_literal), AS("result")))); + auto aggr = ExpectAggregate({sum}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, AggregatonWithListWithAggregationAndGroupBy) { + // Test RETURN sum(2), [sum(3), 42] + AstStorage storage; + auto sum2 = SUM(LITERAL(2)); + auto sum3 = SUM(LITERAL(3)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN(sum2, AS("sum2"), LIST(sum3, group_by_literal), AS("list")))); + auto aggr = ExpectAggregate({sum2, sum3}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MapWithAggregationAndGroupBy) { + // Test RETURN {lit: 42, sum: sum(2)} + AstStorage storage; + FakeDistributedDbAccessor dba; + auto sum = SUM(LITERAL(2)); + auto group_by_literal = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(RETURN( + MAP({storage.GetPropertyIx("sum"), sum}, {storage.GetPropertyIx("lit"), group_by_literal}), AS("result")))); + auto aggr = ExpectAggregate({sum}, {group_by_literal}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, AtomIndexedLabelProperty) { + // Test MATCH (n :label {property: 42, not_indexed: 0}) RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + auto not_indexed = PROPERTY_PAIR("not_indexed"); + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, property.second, 1); + auto node = NODE("n", "label"); + auto lit_42 = LITERAL(42); + std::get<0>(node->properties_)[storage.GetPropertyIx(property.first)] = lit_42; + std::get<0>(node->properties_)[storage.GetPropertyIx(not_indexed.first)] = LITERAL(0); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node)), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, AtomPropertyWhereLabelIndexing) { + // Test MATCH (n {property: 42}) WHERE n.not_indexed AND n:label RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + auto not_indexed = PROPERTY_PAIR("not_indexed"); + dba.SetIndexCount(label, property.second, 0); + auto node = NODE("n"); + auto lit_42 = LITERAL(42); + std::get<0>(node->properties_)[storage.GetPropertyIx(property.first)] = lit_42; + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(node)), + WHERE(AND(PROPERTY_LOOKUP("n", not_indexed), + storage.Create( + IDENT("n"), std::vector{storage.GetLabelIx("label")}))), + RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, WhereIndexedLabelProperty) { + // Test MATCH (n :label) WHERE n.property = 42 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + dba.SetIndexCount(label, property.second, 0); + auto lit_42 = LITERAL(42); + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(EQ(PROPERTY_LOOKUP("n", property), lit_42)), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, BestPropertyIndexed) { + // Test MATCH (n :label) WHERE n.property = 1 AND n.better = 42 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = dba.Property("property"); + // Add a vertex with :label+property combination, so that the best + // :label+better remains empty and thus better choice. + dba.SetIndexCount(label, property, 1); + auto better = PROPERTY_PAIR("better"); + dba.SetIndexCount(label, better.second, 0); + auto lit_42 = LITERAL(42); + auto *query = QUERY( + SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(AND(EQ(PROPERTY_LOOKUP("n", property), LITERAL(1)), EQ(PROPERTY_LOOKUP("n", better), lit_42))), + RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, better, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MultiPropertyIndexScan) { + // Test MATCH (n :label1), (m :label2) WHERE n.prop1 = 1 AND m.prop2 = 2 + // RETURN n, m + FakeDistributedDbAccessor dba; + auto label1 = dba.Label("label1"); + auto label2 = dba.Label("label2"); + auto prop1 = PROPERTY_PAIR("prop1"); + auto prop2 = PROPERTY_PAIR("prop2"); + dba.SetIndexCount(label1, prop1.second, 0); + dba.SetIndexCount(label2, prop2.second, 0); + AstStorage storage; + auto lit_1 = LITERAL(1); + auto lit_2 = LITERAL(2); + auto *query = QUERY(SINGLE_QUERY( + MATCH(PATTERN(NODE("n", "label1")), PATTERN(NODE("m", "label2"))), + WHERE(AND(EQ(PROPERTY_LOOKUP("n", prop1), lit_1), EQ(PROPERTY_LOOKUP("m", prop2), lit_2))), RETURN("n", "m"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label1, prop1, lit_1), + ExpectScanAllByLabelPropertyValue(label2, prop2, lit_2), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, WhereIndexedLabelPropertyRange) { + // Test MATCH (n :label) WHERE n.property REL_OP 42 RETURN n + // REL_OP is one of: `<`, `<=`, `>`, `>=` + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = dba.Property("property"); + dba.SetIndexCount(label, property, 0); + AstStorage storage; + auto lit_42 = LITERAL(42); + auto n_prop = PROPERTY_LOOKUP("n", property); + auto check_planned_range = [&label, &property, &dba](const auto &rel_expr, auto lower_bound, auto upper_bound) { + // Shadow the first storage, so that the query is created in this one. + AstStorage storage; + storage.GetLabelIx("label"); + storage.GetPropertyIx("property"); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(rel_expr), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, + ExpectScanAllByLabelPropertyRange(label, property, lower_bound, upper_bound), ExpectProduce()); + }; + { + // Test relation operators which form an upper bound for range. + std::vector> upper_bound_rel_op{ + std::make_pair(LESS(n_prop, lit_42), Bound::Type::EXCLUSIVE), + std::make_pair(LESS_EQ(n_prop, lit_42), Bound::Type::INCLUSIVE), + std::make_pair(GREATER(lit_42, n_prop), Bound::Type::EXCLUSIVE), + std::make_pair(GREATER_EQ(lit_42, n_prop), Bound::Type::INCLUSIVE)}; + for (const auto &rel_op : upper_bound_rel_op) { + check_planned_range(rel_op.first, std::nullopt, Bound(lit_42, rel_op.second)); + } + } + { + // Test relation operators which form a lower bound for range. + std::vector> lower_bound_rel_op{ + std::make_pair(LESS(lit_42, n_prop), Bound::Type::EXCLUSIVE), + std::make_pair(LESS_EQ(lit_42, n_prop), Bound::Type::INCLUSIVE), + std::make_pair(GREATER(n_prop, lit_42), Bound::Type::EXCLUSIVE), + std::make_pair(GREATER_EQ(n_prop, lit_42), Bound::Type::INCLUSIVE)}; + for (const auto &rel_op : lower_bound_rel_op) { + check_planned_range(rel_op.first, Bound(lit_42, rel_op.second), std::nullopt); + } + } +} + +TYPED_TEST(TestPlanner, WherePreferEqualityIndexOverRange) { + // Test MATCH (n :label) WHERE n.property = 42 AND n.property > 0 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + dba.SetIndexCount(label, property.second, 0); + auto lit_42 = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY( + MATCH(PATTERN(NODE("n", "label"))), + WHERE(AND(EQ(PROPERTY_LOOKUP("n", property), lit_42), GREATER(PROPERTY_LOOKUP("n", property), LITERAL(0)))), + RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, UnableToUsePropertyIndex) { + // Test MATCH (n: label) WHERE n.property = n.property RETURN n + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = dba.Property("property"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, property, 0); + AstStorage storage; + auto *query = + QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(EQ(PROPERTY_LOOKUP("n", property), PROPERTY_LOOKUP("n", property))), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We can only get ScanAllByLabelIndex, because we are comparing properties + // with those on the same node. + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, SecondPropertyIndex) { + // Test MATCH (n :label), (m :label) WHERE m.property = n.property RETURN n + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, dba.Property("property"), 0); + AstStorage storage; + auto n_prop = PROPERTY_LOOKUP("n", property); + auto m_prop = PROPERTY_LOOKUP("m", property); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label")), PATTERN(NODE("m", "label"))), + WHERE(EQ(m_prop, n_prop)), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), + // Note: We are scanning for m, therefore property should equal n_prop. + ExpectScanAllByLabelPropertyValue(label, property, n_prop), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ReturnSumGroupByAll) { + // Test RETURN sum([1,2,3]), all(x in [1] where x = 1) + AstStorage storage; + auto sum = SUM(LIST(LITERAL(1), LITERAL(2), LITERAL(3))); + auto *all = ALL("x", LIST(LITERAL(1)), WHERE(EQ(IDENT("x"), LITERAL(1)))); + auto *query = QUERY(SINGLE_QUERY(RETURN(sum, AS("sum"), all, AS("all")))); + auto aggr = ExpectAggregate({sum}, {all}); + CheckPlan(query, storage, aggr, ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchExpandVariable) { + // Test MATCH (n) -[r *..3]-> (m) RETURN r + AstStorage storage; + auto edge = EDGE_VARIABLE("r"); + edge->upper_bound_ = LITERAL(3); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchExpandVariableNoBounds) { + // Test MATCH (n) -[r *]-> (m) RETURN r + AstStorage storage; + auto edge = EDGE_VARIABLE("r"); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchExpandVariableInlinedFilter) { + // Test MATCH (n) -[r :type * {prop: 42}]-> (m) RETURN r + FakeDistributedDbAccessor dba; + auto type = "type"; + auto prop = PROPERTY_PAIR("prop"); + AstStorage storage; + auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH, {type}); + std::get<0>(edge->properties_)[storage.GetPropertyIx(prop.first)] = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), + ExpectExpandVariable(), // Filter is both inlined and post-expand + ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchExpandVariableNotInlinedFilter) { + // Test MATCH (n) -[r :type * {prop: m.prop}]-> (m) RETURN r + FakeDistributedDbAccessor dba; + auto type = "type"; + auto prop = PROPERTY_PAIR("prop"); + AstStorage storage; + auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH, {type}); + std::get<0>(edge->properties_)[storage.GetPropertyIx(prop.first)] = EQ(PROPERTY_LOOKUP("m", prop), LITERAL(42)); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchExpandVariableTotalWeightSymbol) { + // Test MATCH p = (a {id: 0})-[r* wShortest (e, v | 1) total_weight]->(b) + // RETURN * + FakeDistributedDbAccessor dba; + AstStorage storage; + + auto edge = EDGE_VARIABLE("r", Type::WEIGHTED_SHORTEST_PATH, Direction::BOTH, {}, nullptr, nullptr, nullptr, nullptr, + nullptr, IDENT("total_weight")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("*"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + auto *root = dynamic_cast(&planner.plan()); + + ASSERT_TRUE(root); + + const auto &nes = root->named_expressions_; + EXPECT_TRUE(nes.size() == 4); + + std::vector names(nes.size()); + std::transform(nes.begin(), nes.end(), names.begin(), [](const auto *ne) { return ne->name_; }); + + EXPECT_TRUE(root->named_expressions_.size() == 4); + EXPECT_TRUE(memgraph::utils::Contains(names, "m")); + EXPECT_TRUE(memgraph::utils::Contains(names, "n")); + EXPECT_TRUE(memgraph::utils::Contains(names, "r")); + EXPECT_TRUE(memgraph::utils::Contains(names, "total_weight")); +} + +TYPED_TEST(TestPlanner, UnwindMatchVariable) { + // Test UNWIND [1,2,3] AS depth MATCH (n) -[r*d]-> (m) RETURN r + AstStorage storage; + auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::OUT); + edge->lower_bound_ = IDENT("d"); + edge->upper_bound_ = IDENT("d"); + auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1), LITERAL(2), LITERAL(3)), AS("d")), + MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); + CheckPlan(query, storage, ExpectUnwind(), ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchBfs) { + // Test MATCH (n) -[r:type *..10 (r, n|n)]-> (m) RETURN r + FakeDistributedDbAccessor dba; + AstStorage storage; + auto edge_type = storage.GetEdgeTypeIx("type"); + auto *bfs = + storage.Create(IDENT("r"), memgraph::query::EdgeAtom::Type::BREADTH_FIRST, + Direction::OUT, std::vector{edge_type}); + bfs->filter_lambda_.inner_edge = IDENT("r"); + bfs->filter_lambda_.inner_node = IDENT("n"); + bfs->filter_lambda_.expression = IDENT("n"); + bfs->upper_bound_ = LITERAL(10); + auto *as_r = NEXPR("r", IDENT("r")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), bfs, NODE("m"))), RETURN(as_r))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpandBfs(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchDoubleScanToExpandExisting) { + // Test MATCH (n) -[r]- (m :label) RETURN r + FakeDistributedDbAccessor dba; + auto label = "label"; + dba.SetIndexCount(dba.Label(label), 0); + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m", label))), RETURN("r"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect 2x ScanAll and then Expand, since we are guessing that is + // faster (due to low label index vertex count). + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectScanAllByLabel(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchScanToExpand) { + // Test MATCH (n) -[r]- (m :label {property: 1}) RETURN r + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = dba.Property("property"); + // Fill vertices to the max + 1. + dba.SetIndexCount(label, property, FLAGS_query_vertex_count_to_expand_existing + 1); + dba.SetIndexCount(label, FLAGS_query_vertex_count_to_expand_existing + 1); + AstStorage storage; + auto node_m = NODE("m", "label"); + std::get<0>(node_m->properties_)[storage.GetPropertyIx("property")] = LITERAL(1); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), node_m)), RETURN("r"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + // We expect 1x ScanAll and then Expand, since we are guessing that + // is faster (due to high label index vertex count). + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, MatchWhereAndSplit) { + // Test MATCH (n) -[r]- (m) WHERE n.prop AND r.prop RETURN m + FakeDistributedDbAccessor dba; + auto prop = PROPERTY_PAIR("prop"); + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), + WHERE(AND(PROPERTY_LOOKUP("n", prop), PROPERTY_LOOKUP("r", prop))), RETURN("m"))); + // We expect `n.prop` filter right after scanning `n`. + CheckPlan(query, storage, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ReturnAsteriskOmitsLambdaSymbols) { + // Test MATCH (n) -[r* (ie, in | true)]- (m) RETURN * + AstStorage storage; + auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH); + edge->filter_lambda_.inner_edge = IDENT("ie"); + edge->filter_lambda_.inner_node = IDENT("in"); + edge->filter_lambda_.expression = LITERAL(true); + auto ret = storage.Create(); + ret->body_.all_identifiers = true; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), ret)); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + auto *produce = dynamic_cast(&planner.plan()); + ASSERT_TRUE(produce); + std::vector outputs; + for (const auto &output_symbol : produce->OutputSymbols(symbol_table)) { + outputs.emplace_back(output_symbol.name()); + } + // We expect `*` expanded to `n`, `r` and `m`. + EXPECT_EQ(outputs.size(), 3); + for (const auto &name : {"n", "r", "m"}) { + EXPECT_TRUE(memgraph::utils::Contains(outputs, name)); + } +} + +TYPED_TEST(TestPlanner, FilterRegexMatchIndex) { + // Test MATCH (n :label) WHERE n.prop =~ "regex" RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + auto label = dba.Label("label"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, prop, 0); + auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(regex_match), RETURN("n"))); + // We expect that we use index by property range where lower bound is an empty + // string. Filter must still remain in place, because we don't have regex + // based index. + Bound lower_bound(LITERAL(""), Bound::Type::INCLUSIVE); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyRange(label, prop, lower_bound, std::nullopt), + ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, FilterRegexMatchPreferEqualityIndex) { + // Test MATCH (n :label) WHERE n.prop =~ "regex" AND n.prop = 42 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = PROPERTY_PAIR("prop"); + auto label = dba.Label("label"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, prop.second, 0); + auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); + auto *lit_42 = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(AND(regex_match, EQ(PROPERTY_LOOKUP("n", prop), lit_42))), RETURN("n"))); + // We expect that we use index by property value equal to 42, because that's + // much better than property range for regex matching. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, prop, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, FilterRegexMatchPreferEqualityIndex2) { + // Test MATCH (n :label) + // WHERE n.prop =~ "regex" AND n.prop = 42 AND n.prop > 0 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = PROPERTY_PAIR("prop"); + auto label = dba.Label("label"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, prop.second, 0); + auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); + auto *lit_42 = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(AND(AND(regex_match, EQ(PROPERTY_LOOKUP("n", prop), lit_42)), + GREATER(PROPERTY_LOOKUP("n", prop), LITERAL(0)))), + RETURN("n"))); + // We expect that we use index by property value equal to 42, because that's + // much better than property range. + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, prop, lit_42), ExpectFilter(), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, FilterRegexMatchPreferRangeIndex) { + // Test MATCH (n :label) WHERE n.prop =~ "regex" AND n.prop > 42 RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto prop = dba.Property("prop"); + auto label = dba.Label("label"); + dba.SetIndexCount(label, 0); + dba.SetIndexCount(label, prop, 0); + auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); + auto *lit_42 = LITERAL(42); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(AND(regex_match, GREATER(PROPERTY_LOOKUP("n", prop), lit_42))), RETURN("n"))); + // We expect that we use index by property range on a concrete value (42), as + // it is much better than using a range from empty string for regex matching. + Bound lower_bound(lit_42, Bound::Type::EXCLUSIVE); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyRange(label, prop, lower_bound, std::nullopt), + ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CallProcedureStandalone) { + // Test CALL proc(1,2,3) YIELD field AS result + AstStorage storage; + auto *ast_call = storage.Create(); + ast_call->procedure_name_ = "proc"; + ast_call->arguments_ = {LITERAL(1), LITERAL(2), LITERAL(3)}; + ast_call->result_fields_ = {"field"}; + ast_call->result_identifiers_ = {IDENT("result")}; + auto *query = QUERY(SINGLE_QUERY(ast_call)); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + std::vector result_syms; + result_syms.reserve(ast_call->result_identifiers_.size()); + for (const auto *ident : ast_call->result_identifiers_) { + result_syms.push_back(symbol_table.at(*ident)); + } + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan( + planner.plan(), symbol_table, + ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms)); +} + +TYPED_TEST(TestPlanner, CallProcedureAfterScanAll) { + // Test MATCH (n) CALL proc(n) YIELD field AS result RETURN result + AstStorage storage; + auto *ast_call = storage.Create(); + ast_call->procedure_name_ = "proc"; + ast_call->arguments_ = {IDENT("n")}; + ast_call->result_fields_ = {"field"}; + ast_call->result_identifiers_ = {IDENT("result")}; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), ast_call, RETURN("result"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + std::vector result_syms; + result_syms.reserve(ast_call->result_identifiers_.size()); + for (const auto *ident : ast_call->result_identifiers_) { + result_syms.push_back(symbol_table.at(*ident)); + } + FakeDistributedDbAccessor dba; + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), + ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms), + ExpectProduce()); +} + +TYPED_TEST(TestPlanner, CallProcedureBeforeScanAll) { + // Test CALL proc() YIELD field MATCH (n) WHERE n.prop = field RETURN n + AstStorage storage; + auto *ast_call = storage.Create(); + ast_call->procedure_name_ = "proc"; + ast_call->result_fields_ = {"field"}; + ast_call->result_identifiers_ = {IDENT("field")}; + FakeDistributedDbAccessor dba; + auto property = dba.Property("prop"); + auto *query = QUERY(SINGLE_QUERY(ast_call, MATCH(PATTERN(NODE("n"))), + WHERE(EQ(PROPERTY_LOOKUP("n", property), IDENT("field"))), RETURN("n"))); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + std::vector result_syms; + result_syms.reserve(ast_call->result_identifiers_.size()); + for (const auto *ident : ast_call->result_identifiers_) { + result_syms.push_back(symbol_table.at(*ident)); + } + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, + ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms), + ExpectScanAll(), ExpectFilter(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ScanAllById) { + // Test MATCH (n) WHERE id(n) = 42 RETURN n + AstStorage storage; + auto *query = + QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WHERE(EQ(FN("id", IDENT("n")), LITERAL(42))), RETURN("n"))); + CheckPlan(query, storage, ExpectScanAllById(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, ScanAllByIdExpandToExisting) { + // Test MATCH (n)-[r]-(m) WHERE id(m) = 42 RETURN r + AstStorage storage; + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), + WHERE(EQ(FN("id", IDENT("m")), LITERAL(42))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), ExpectScanAllById(), ExpectExpand(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, BfsToExisting) { + // Test MATCH (n)-[r *bfs]-(m) WHERE id(m) = 42 RETURN r + AstStorage storage; + auto *bfs = storage.Create(IDENT("r"), memgraph::query::EdgeAtom::Type::BREADTH_FIRST, + Direction::BOTH); + bfs->filter_lambda_.inner_edge = IDENT("ie"); + bfs->filter_lambda_.inner_node = IDENT("in"); + bfs->filter_lambda_.expression = LITERAL(true); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), bfs, NODE("m"))), + WHERE(EQ(FN("id", IDENT("m")), LITERAL(42))), RETURN("r"))); + CheckPlan(query, storage, ExpectScanAll(), ExpectScanAllById(), ExpectExpandBfs(), ExpectProduce()); +} + +TYPED_TEST(TestPlanner, LabelPropertyInListValidOptimization) { + // Test MATCH (n:label) WHERE n.property IN ['a'] RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + auto *lit_list_a = LIST(LITERAL('a')); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(IN_LIST(PROPERTY_LOOKUP("n", property), lit_list_a)), RETURN("n"))); + { + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); + } + { + dba.SetIndexCount(label, 1); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); + } + { + dba.SetIndexCount(label, property.second, 1); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectUnwind(), + ExpectScanAllByLabelPropertyValue(label, property, lit_list_a), ExpectProduce()); + } +} + +TYPED_TEST(TestPlanner, LabelPropertyInListWhereLabelPropertyOnLeftNotListOnRight) { + // Test MATCH (n:label) WHERE n.property IN 'a' RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + auto *lit_a = LITERAL('a'); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(IN_LIST(PROPERTY_LOOKUP("n", property), lit_a)), RETURN("n"))); + { + dba.SetIndexCount(label, property.second, 1); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); + } +} + +TYPED_TEST(TestPlanner, LabelPropertyInListWhereLabelPropertyOnRight) { + // Test MATCH (n:label) WHERE ['a'] IN n.property RETURN n + AstStorage storage; + FakeDistributedDbAccessor dba; + auto label = dba.Label("label"); + auto property = PROPERTY_PAIR("property"); + auto *lit_list_a = LIST(LITERAL('a')); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), + WHERE(IN_LIST(lit_list_a, PROPERTY_LOOKUP("n", property))), RETURN("n"))); + { + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); + } + { + dba.SetIndexCount(label, 1); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); + } + { + dba.SetIndexCount(label, property.second, 1); + auto symbol_table = memgraph::query::MakeSymbolTable(query); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); + } +} + +TYPED_TEST(TestPlanner, Foreach) { + AstStorage storage; + FakeDistributedDbAccessor dba; + { + auto *i = NEXPR("i", IDENT("i")); + auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {CREATE(PATTERN(NODE("n")))}))); + auto create = ExpectCreateNode(); + std::list updates{&create}; + std::list input; + CheckPlan(query, storage, ExpectForeach(input, updates)); + } + { + auto *i = NEXPR("i", IDENT("i")); + auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {DELETE(IDENT("i"))}))); + auto del = ExpectDelete(); + std::list updates{&del}; + std::list input; + CheckPlan(query, storage, ExpectForeach({input}, updates)); + } + { + auto prop = dba.Property("prop"); + auto *i = NEXPR("i", IDENT("i")); + auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {SET(PROPERTY_LOOKUP("i", prop), LITERAL(10))}))); + auto set_prop = ExpectSetProperty(); + std::list updates{&set_prop}; + std::list input; + CheckPlan(query, storage, ExpectForeach({input}, updates)); + } + { + auto *i = NEXPR("i", IDENT("i")); + auto *j = NEXPR("j", IDENT("j")); + auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {FOREACH(j, {CREATE(PATTERN(NODE("n"))), DELETE(IDENT("i"))})}))); + auto create = ExpectCreateNode(); + auto del = ExpectDelete(); + std::list input; + std::list nested_updates{{&create, &del}}; + auto nested_foreach = ExpectForeach(input, nested_updates); + std::list updates{&nested_foreach}; + CheckPlan(query, storage, ExpectForeach(input, updates)); + } + { + auto *i = NEXPR("i", IDENT("i")); + auto *j = NEXPR("j", IDENT("j")); + auto create = ExpectCreateNode(); + std::list empty; + std::list updates{&create}; + auto input_op = ExpectForeach(empty, updates); + std::list input{&input_op}; + auto *query = + QUERY(SINGLE_QUERY(FOREACH(i, {CREATE(PATTERN(NODE("n")))}), FOREACH(j, {CREATE(PATTERN(NODE("n")))}))); + CheckPlan(query, storage, ExpectForeach(input, updates)); + } +} +*/ + +} // namespace From 7e8b4921b40c03e5d91e3756e167f7408dcf6088 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Wed, 30 Nov 2022 13:16:04 +0100 Subject: [PATCH 005/251] Make query_v2_plan unit tests available again. The query_v2_plan unit tests were needed in order to properly test if the created logical plan of the new operator work properly. In order to achieve this v2 versions of the several files were created, where the old utilities were replaced with new ones, like query::v2 and storage::v3. A new fake db accessor was also created in order to be able to test the ScanAllByPrimaryKey operator. --- src/query/v2/plan/pretty_print.cpp | 21 + src/query/v2/plan/pretty_print.hpp | 2 + src/query/v2/plan/rewrite/index_lookup.hpp | 39 +- src/query/v2/plan/vertex_count_cache.hpp | 12 +- tests/unit/CMakeLists.txt | 4 +- tests/unit/query_common.hpp | 247 ------- tests/unit/query_plan_checker_v2.hpp | 86 ++- tests/unit/query_v2_common.hpp | 603 ++++++++++++++++++ .../{query_plan_v2.cpp => query_v2_plan.cpp} | 123 ++-- 9 files changed, 815 insertions(+), 322 deletions(-) create mode 100644 tests/unit/query_v2_common.hpp rename tests/unit/{query_plan_v2.cpp => query_v2_plan.cpp} (95%) diff --git a/src/query/v2/plan/pretty_print.cpp b/src/query/v2/plan/pretty_print.cpp index 76cc6ac26..4be8f4884 100644 --- a/src/query/v2/plan/pretty_print.cpp +++ b/src/query/v2/plan/pretty_print.cpp @@ -86,6 +86,14 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) { return true; } +bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByPrimaryKey &op) { + WithPrintLn([&](auto &out) { + out << "* ScanAllByPrimaryKey" + << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")"; + }); + return true; +} + bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) { WithPrintLn([&](auto &out) { *out_ << "* Expand (" << op.input_symbol_.name() << ")" @@ -480,6 +488,19 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) { return false; } +bool PlanToJsonVisitor::PreVisit(ScanAllByPrimaryKey &op) { + json self; + self["name"] = "ScanAllByPrimaryKey"; + self["label"] = ToJson(op.label_, *request_manager_); + self["output_symbol"] = ToJson(op.output_symbol_); + + op.input_->Accept(*this); + self["input"] = PopOutput(); + + output_ = std::move(self); + return false; +} + bool PlanToJsonVisitor::PreVisit(CreateNode &op) { json self; self["name"] = "CreateNode"; diff --git a/src/query/v2/plan/pretty_print.hpp b/src/query/v2/plan/pretty_print.hpp index 66fa31556..c1c84db40 100644 --- a/src/query/v2/plan/pretty_print.hpp +++ b/src/query/v2/plan/pretty_print.hpp @@ -68,6 +68,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelProperty &) override; + bool PreVisit(query::v2::plan::ScanAllByPrimaryKey &) override; bool PreVisit(Expand &) override; bool PreVisit(ExpandVariable &) override; @@ -195,6 +196,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelProperty &) override; + bool PreVisit(ScanAllByPrimaryKey &) override; bool PreVisit(Produce &) override; bool PreVisit(Accumulate &) override; diff --git a/src/query/v2/plan/rewrite/index_lookup.hpp b/src/query/v2/plan/rewrite/index_lookup.hpp index 41e24d2b6..1a4a4160c 100644 --- a/src/query/v2/plan/rewrite/index_lookup.hpp +++ b/src/query/v2/plan/rewrite/index_lookup.hpp @@ -273,6 +273,16 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { return true; } + bool PreVisit(ScanAllByPrimaryKey &op) override { + prev_ops_.push_back(&op); + return true; + } + + bool PostVisit(ScanAllByPrimaryKey &) override { + prev_ops_.pop_back(); + return true; + } + bool PreVisit(ConstructNamedPath &op) override { prev_ops_.push_back(&op); return true; @@ -480,6 +490,12 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { storage::v3::PropertyId GetProperty(PropertyIx prop) { return db_->NameToProperty(prop.name); } + void EraseLabelFilters(const memgraph::query::v2::Symbol &node_symbol, memgraph::query::v2::LabelIx prim_label) { + std::vector removed_expressions; + filters_.EraseLabelFilter(node_symbol, prim_label, &removed_expressions); + filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end()); + } + std::optional FindBestLabelIndex(const std::unordered_set &labels) { MG_ASSERT(!labels.empty(), "Trying to find the best label without any labels."); std::optional best_label; @@ -564,19 +580,28 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { // First, try to see if we can find a vertex based on the possibly // supplied primary key. auto property_filters = filters_.PropertyFilters(node_symbol); - storage::v3::LabelId prim_label; - std::vector primary_key; + query::v2::LabelIx prim_label; + std::vector> primary_key; if (!property_filters.empty()) { for (const auto &label : labels) { if (db_->LabelIndexExists(GetLabel(label))) { - prim_label = GetLabel(label); - primary_key = db_->ExtractPrimaryKey(prim_label, property_filters); + prim_label = label; + primary_key = db_->ExtractPrimaryKey(GetLabel(prim_label), property_filters); break; } } if (!primary_key.empty()) { - return std::make_unique(input, node_symbol, prim_label, primary_key); + // Mark the expressions so they won't be used for an additional, unnecessary filter. + for (const auto &pk : primary_key) { + filter_exprs_for_removal_.insert(pk.first); + filters_.EraseFilter(pk.second); + } + EraseLabelFilters(node_symbol, prim_label); + std::vector pk_expressions; + std::transform(primary_key.begin(), primary_key.end(), std::back_inserter(pk_expressions), + [](const auto &exp) { return exp.first; }); + return std::make_unique(input, node_symbol, GetLabel(prim_label), pk_expressions); } } @@ -593,9 +618,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { filter_exprs_for_removal_.insert(found_index->filter.expression); } filters_.EraseFilter(found_index->filter); - std::vector removed_expressions; - filters_.EraseLabelFilter(node_symbol, found_index->label, &removed_expressions); - filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end()); + EraseLabelFilters(node_symbol, found_index->label); if (prop_filter.lower_bound_ || prop_filter.upper_bound_) { return std::make_unique( input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_), diff --git a/src/query/v2/plan/vertex_count_cache.hpp b/src/query/v2/plan/vertex_count_cache.hpp index 70b8d324b..120e434ea 100644 --- a/src/query/v2/plan/vertex_count_cache.hpp +++ b/src/query/v2/plan/vertex_count_cache.hpp @@ -58,9 +58,9 @@ class VertexCountCache { bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; } - std::vector ExtractPrimaryKey(storage::v3::LabelId label, - std::vector property_filters) { - std::vector pk; + std::vector> ExtractPrimaryKey( + storage::v3::LabelId label, std::vector property_filters) { + std::vector> pk; const auto schema = shard_request_manager_->GetSchemaForLabel(label); std::vector schema_properties; @@ -72,11 +72,13 @@ class VertexCountCache { for (const auto &property_filter : property_filters) { const auto &property_id = NameToProperty(property_filter.property_filter->property_.name); if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) { - pk.push_back(property_filter.expression); + pk.emplace_back(std::make_pair(property_filter.expression, property_filter)); } } - return pk.size() == schema_properties.size() ? pk : std::vector{}; + return pk.size() == schema_properties.size() + ? pk + : std::vector>{}; } msgs::ShardRequestManagerInterface *shard_request_manager_; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index d9f9df875..123a448f5 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -93,8 +93,8 @@ target_link_libraries(${test_prefix}query_expression_evaluator mg-query) add_unit_test(query_plan.cpp) target_link_libraries(${test_prefix}query_plan mg-query) -add_unit_test(query_plan_v2.cpp) -target_link_libraries(${test_prefix}query_plan_v2 mg-query-v2) +add_unit_test(query_v2_plan.cpp) +target_link_libraries(${test_prefix}query_v2_plan mg-query-v2) add_unit_test(query_plan_accumulate_aggregate.cpp) target_link_libraries(${test_prefix}query_plan_accumulate_aggregate mg-query) diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index 4697124ef..f7fe6e805 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -84,38 +84,14 @@ struct OrderBy { std::vector expressions; }; -// new stuff begin - -struct OrderByv2 { - std::vector expressions; -}; - -// new stuff end - struct Skip { Expression *expression = nullptr; }; -// new stuff begin - -struct Skipv2 { - query::v2::Expression *expression = nullptr; -}; - -// new stuff end - struct Limit { Expression *expression = nullptr; }; -// new stuff begin - -struct Limitv2 { - query::v2::Expression *expression = nullptr; -}; - -// new stuff end - struct OnMatch { std::vector set; }; @@ -182,42 +158,6 @@ auto GetPropertyLookup(AstStorage &storage, TDbAccessor &, Expression *expr, return storage.Create(expr, storage.GetPropertyIx(prop_pair.first)); } -// new stuff begin - -template -auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, const std::string &name, - memgraph::storage::v3::PropertyId property) { - return storage.Create(storage.Create(name), - storage.GetPropertyIx(dba.PropertyToName(property))); -} - -template -auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, - memgraph::query::v2::Expression *expr, memgraph::storage::v3::PropertyId property) { - return storage.Create(expr, storage.GetPropertyIx(dba.PropertyToName(property))); -} - -template -auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &dba, - memgraph::query::v2::Expression *expr, const std::string &property) { - return storage.Create(expr, storage.GetPropertyIx(property)); -} - -template -auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &, const std::string &name, - const std::pair &prop_pair) { - return storage.Create(storage.Create(name), - storage.GetPropertyIx(prop_pair.first)); -} - -template -auto GetPropertyLookup(memgraph::query::v2::AstStorage &storage, TDbAccessor &, memgraph::query::v2::Expression *expr, - const std::pair &prop_pair) { - return storage.Create(expr, storage.GetPropertyIx(prop_pair.first)); -} - -// new stuff end - /// Create an EdgeAtom with given name, direction and edge_type. /// /// Name is used to create the Identifier which is assigned to the edge. @@ -276,15 +216,6 @@ auto GetNode(AstStorage &storage, const std::string &name, std::optional label = std::nullopt) { - auto node = storage.Create(storage.Create(name)); - if (label) node->labels_.emplace_back(storage.GetLabelIx(*label)); - return node; -} -// new stuff end - /// Create a Pattern with given atoms. auto GetPattern(AstStorage &storage, std::vector atoms) { auto pattern = storage.Create(); @@ -301,26 +232,6 @@ auto GetPattern(AstStorage &storage, const std::string &name, std::vector atoms) { - auto pattern = storage.Create(); - pattern->identifier_ = storage.Create(memgraph::utils::RandomString(20), false); - pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); - return pattern; -} - -/// Create a Pattern with given name and atoms. -auto GetPattern(memgraph::query::v2::AstStorage &storage, const std::string &name, - std::vector atoms) { - auto pattern = storage.Create(); - pattern->identifier_ = storage.Create(name, true); - pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); - return pattern; -} - -// new stuff end - /// This function fills an AST node which with given patterns. /// /// The function is most commonly used to create Match and Create clauses. @@ -330,16 +241,6 @@ auto GetWithPatterns(TWithPatterns *with_patterns, std::vector patter return with_patterns; } -// new stuff begin - -template -auto GetWithPatterns(TWithPatterns *with_patterns, std::vector patterns) { - with_patterns->patterns_.insert(with_patterns->patterns_.begin(), patterns.begin(), patterns.end()); - return with_patterns; -} - -// new stuff end - /// Create a query with given clauses. auto GetSingleQuery(SingleQuery *single_query, Clause *clause) { @@ -375,45 +276,6 @@ auto GetSingleQuery(SingleQuery *single_query, Clause *clause, T *...clauses) { return GetSingleQuery(single_query, clauses...); } -// new stuff begin - -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Clause *clause) { - single_query->clauses_.emplace_back(clause); - return single_query; -} -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Match *match, query::v2::Where *where) { - match->where_ = where; - single_query->clauses_.emplace_back(match); - return single_query; -} -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::With *with, query::v2::Where *where) { - with->where_ = where; - single_query->clauses_.emplace_back(with); - return single_query; -} -template -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Match *match, query::v2::Where *where, - T *...clauses) { - match->where_ = where; - single_query->clauses_.emplace_back(match); - return GetSingleQuery(single_query, clauses...); -} -template -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::With *with, query::v2::Where *where, - T *...clauses) { - with->where_ = where; - single_query->clauses_.emplace_back(with); - return GetSingleQuery(single_query, clauses...); -} - -template -auto GetSingleQuery(query::v2::SingleQuery *single_query, query::v2::Clause *clause, T *...clauses) { - single_query->clauses_.emplace_back(clause); - return GetSingleQuery(single_query, clauses...); -} - -// new stuff end - auto GetCypherUnion(CypherUnion *cypher_union, SingleQuery *single_query) { cypher_union->single_query_ = single_query; return cypher_union; @@ -433,24 +295,6 @@ auto GetQuery(AstStorage &storage, SingleQuery *single_query, T *...cypher_union return query; } -// new stuff begin - -auto GetQuery(query::v2::AstStorage &storage, query::v2::SingleQuery *single_query) { - auto *query = storage.Create(); - query->single_query_ = single_query; - return query; -} - -template -auto GetQuery(query::v2::AstStorage &storage, query::v2::SingleQuery *single_query, T *...cypher_unions) { - auto *query = storage.Create(); - query->single_query_ = single_query; - query->cypher_unions_ = std::vector{cypher_unions...}; - return query; -} - -// new stuff end - // Helper functions for constructing RETURN and WITH clauses. void FillReturnBody(AstStorage &, ReturnBody &body, NamedExpression *named_expr) { body.named_expressions.emplace_back(named_expr); @@ -514,80 +358,6 @@ void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &na FillReturnBody(storage, body, rest...); } -// new stuff begin - -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, query::v2::NamedExpression *named_expr) { - body.named_expressions.emplace_back(named_expr); -} -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name) { - if (name == "*") { - body.all_identifiers = true; - } else { - auto *ident = storage.Create(name); - auto *named_expr = storage.Create(name, ident); - body.named_expressions.emplace_back(named_expr); - } -} -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, Limitv2 limit) { - body.limit = limit.expression; -} -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, Skipv2 skip, Limitv2 limit = Limitv2{}) { - body.skip = skip.expression; - body.limit = limit.expression; -} -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, OrderByv2 order_by, - Limitv2 limit = Limitv2{}) { - body.order_by = order_by.expressions; - body.limit = limit.expression; -} -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, OrderByv2 order_by, Skipv2 skip, - Limitv2 limit = Limitv2{}) { - body.order_by = order_by.expressions; - body.skip = skip.expression; - body.limit = limit.expression; -} -void FillReturnBody(query::v2::AstStorage &, query::v2::ReturnBody &body, query::v2::Expression *expr, - query::v2::NamedExpression *named_expr) { - // This overload supports `RETURN(expr, AS(name))` construct, since - // NamedExpression does not inherit Expression. - named_expr->expression_ = expr; - body.named_expressions.emplace_back(named_expr); -} -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, - query::v2::NamedExpression *named_expr) { - named_expr->expression_ = storage.Create(name); - body.named_expressions.emplace_back(named_expr); -} -template -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, query::v2::Expression *expr, - query::v2::NamedExpression *named_expr, T... rest) { - named_expr->expression_ = expr; - body.named_expressions.emplace_back(named_expr); - FillReturnBody(storage, body, rest...); -} -template -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, query::v2::NamedExpression *named_expr, - T... rest) { - body.named_expressions.emplace_back(named_expr); - FillReturnBody(storage, body, rest...); -} -template -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, - query::v2::NamedExpression *named_expr, T... rest) { - named_expr->expression_ = storage.Create(name); - body.named_expressions.emplace_back(named_expr); - FillReturnBody(storage, body, rest...); -} -template -void FillReturnBody(query::v2::AstStorage &storage, query::v2::ReturnBody &body, const std::string &name, T... rest) { - auto *ident = storage.Create(name); - auto *named_expr = storage.Create(name, ident); - body.named_expressions.emplace_back(named_expr); - FillReturnBody(storage, body, rest...); -} - -// new stuff end - /// Create the return clause with given expressions. /// /// The supported expression combination of arguments is: @@ -609,18 +379,6 @@ auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { return ret; } -// new stuff begin - -template -auto GetReturn(query::v2::AstStorage &storage, bool distinct, T... exprs) { - auto ret = storage.Create(); - ret->body_.distinct = distinct; - FillReturnBody(storage, ret->body_, exprs...); - return ret; -} - -// new stuff end - /// Create the with clause with given expressions. /// /// The supported expression combination is the same as for @c GetReturn. @@ -739,10 +497,7 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec memgraph::query::test_common::GetWithPatterns(storage.Create(true), {__VA_ARGS__}) #define MATCH(...) \ memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) -#define MATCH_V2(...) \ - memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) #define WHERE(expr) storage.Create((expr)) -#define WHERE_V2(expr) storage.Create((expr)) #define CREATE(...) \ memgraph::query::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) #define IDENT(...) storage.Create(__VA_ARGS__) @@ -789,8 +544,6 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec std::vector{(property)}) #define QUERY(...) memgraph::query::test_common::GetQuery(storage, __VA_ARGS__) #define SINGLE_QUERY(...) memgraph::query::test_common::GetSingleQuery(storage.Create(), __VA_ARGS__) -#define SINGLE_QUERY_V2(...) \ - memgraph::query::test_common::GetSingleQuery(storage.Create(), __VA_ARGS__) #define UNION(...) memgraph::query::test_common::GetCypherUnion(storage.Create(true), __VA_ARGS__) #define UNION_ALL(...) memgraph::query::test_common::GetCypherUnion(storage.Create(false), __VA_ARGS__) #define FOREACH(...) memgraph::query::test_common::GetForeach(storage, __VA_ARGS__) diff --git a/tests/unit/query_plan_checker_v2.hpp b/tests/unit/query_plan_checker_v2.hpp index 6a6a12680..1ad1e712a 100644 --- a/tests/unit/query_plan_checker_v2.hpp +++ b/tests/unit/query_plan_checker_v2.hpp @@ -61,6 +61,7 @@ class PlanChecker : public virtual HierarchicalLogicalOperatorVisitor { PRE_VISIT(ScanAllByLabelPropertyValue); PRE_VISIT(ScanAllByLabelPropertyRange); PRE_VISIT(ScanAllByLabelProperty); + PRE_VISIT(ScanAllByPrimaryKey); PRE_VISIT(Expand); PRE_VISIT(ExpandVariable); PRE_VISIT(Filter); @@ -270,25 +271,25 @@ using ExpectDistinct = OpChecker; // const std::list &optional_; // }; -// class ExpectScanAllByLabelPropertyValue : public OpChecker { -// public: -// ExpectScanAllByLabelPropertyValue(memgraph::storage::LabelId label, -// const std::pair &prop_pair, -// memgraph::query::Expression *expression) -// : label_(label), property_(prop_pair.second), expression_(expression) {} +class ExpectScanAllByLabelPropertyValue : public OpChecker { + public: + ExpectScanAllByLabelPropertyValue(memgraph::storage::v3::LabelId label, + const std::pair &prop_pair, + memgraph::query::v2::Expression *expression) + : label_(label), property_(prop_pair.second), expression_(expression) {} -// void ExpectOp(ScanAllByLabelPropertyValue &scan_all, const SymbolTable &) override { -// EXPECT_EQ(scan_all.label_, label_); -// EXPECT_EQ(scan_all.property_, property_); -// // TODO: Proper expression equality -// EXPECT_EQ(typeid(scan_all.expression_).hash_code(), typeid(expression_).hash_code()); -// } + void ExpectOp(ScanAllByLabelPropertyValue &scan_all, const SymbolTable &) override { + EXPECT_EQ(scan_all.label_, label_); + EXPECT_EQ(scan_all.property_, property_); + // TODO: Proper expression equality + EXPECT_EQ(typeid(scan_all.expression_).hash_code(), typeid(expression_).hash_code()); + } -// private: -// memgraph::storage::LabelId label_; -// memgraph::storage::PropertyId property_; -// memgraph::query::Expression *expression_; -// }; + private: + memgraph::storage::v3::LabelId label_; + memgraph::storage::v3::PropertyId property_; + memgraph::query::v2::Expression *expression_; +}; // class ExpectScanAllByLabelPropertyRange : public OpChecker { // public: @@ -536,12 +537,53 @@ class FakeDistributedDbAccessor { } memgraph::storage::v3::PropertyId NameToProperty(const std::string &name) { - return storage::v3::PropertyId::FromUint(0); + auto find_in_prim_properties = primary_properties_.find(name); + if (find_in_prim_properties != primary_properties_.end()) { + return find_in_prim_properties->second; + } + auto find_in_secondary_properties = secondary_properties_.find(name); + if (find_in_secondary_properties != secondary_properties_.end()) { + return find_in_secondary_properties->second; + } + + MG_ASSERT(false, "The property does not exist as a primary or a secondary property."); + return memgraph::storage::v3::PropertyId::FromUint(0); } - std::vector ExtractPrimaryKey(storage::v3::LabelId label, - std::vector property_filters) { - return std::vector{}; + std::vector> ExtractPrimaryKey( + storage::v3::LabelId label, std::vector property_filters) { + MG_ASSERT(schemas_.contains(label), + "You did not specify the Schema for this label! Use FakeDistributedDbAccessor::CreateSchema(...)."); + + std::vector> pk; + const auto schema = GetSchemaForLabel(label); + + std::vector schema_properties; + schema_properties.reserve(schema.size()); + + std::transform(schema.begin(), schema.end(), std::back_inserter(schema_properties), + [](const auto &schema_elem) { return schema_elem; }); + + for (const auto &property_filter : property_filters) { + const auto &property_id = NameToProperty(property_filter.property_filter->property_.name); + if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) { + pk.emplace_back(std::make_pair(property_filter.expression, property_filter)); + } + } + + return pk.size() == schema_properties.size() + ? pk + : std::vector>{}; + } + + std::vector GetSchemaForLabel(storage::v3::LabelId label) { + return schemas_.at(label); + } + + void CreateSchema(const memgraph::storage::v3::LabelId primary_label, + const std::vector &schemas_types) { + MG_ASSERT(!schemas_.contains(primary_label), "You already created the schema for this label!"); + schemas_.emplace(primary_label, schemas_types); } private: @@ -554,6 +596,8 @@ class FakeDistributedDbAccessor { std::unordered_map label_index_; std::vector> label_property_index_; + + std::unordered_map> schemas_; }; } // namespace memgraph::query::v2::plan diff --git a/tests/unit/query_v2_common.hpp b/tests/unit/query_v2_common.hpp new file mode 100644 index 000000000..85905cb22 --- /dev/null +++ b/tests/unit/query_v2_common.hpp @@ -0,0 +1,603 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +/// @file +/// This file provides macros for easier construction of openCypher query AST. +/// The usage of macros is very similar to how one would write openCypher. For +/// example: +/// +/// AstStorage storage; // Macros rely on storage being in scope. +/// // PROPERTY_LOOKUP and PROPERTY_PAIR macros +/// // rely on a DbAccessor *reference* named dba. +/// database::GraphDb db; +/// auto dba_ptr = db.Access(); +/// auto &dba = *dba_ptr; +/// +/// QUERY(MATCH(PATTERN(NODE("n"), EDGE("e"), NODE("m"))), +/// WHERE(LESS(PROPERTY_LOOKUP("e", edge_prop), LITERAL(3))), +/// RETURN(SUM(PROPERTY_LOOKUP("m", prop)), AS("sum"), +/// ORDER_BY(IDENT("sum")), +/// SKIP(ADD(LITERAL(1), LITERAL(2))))); +/// +/// Each of the macros is accompanied by a function. The functions use overload +/// resolution and template magic to provide a type safe way of constructing +/// queries. Although the functions can be used by themselves, it is more +/// convenient to use the macros. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "query/frontend/ast/pretty_print.hpp" // not sure if that is ok... +#include "query/v2/frontend/ast/ast.hpp" +#include "storage/v3/id_types.hpp" +#include "utils/string.hpp" + +#include "query/v2/frontend/ast/ast.hpp" + +namespace memgraph::query::v2 { + +namespace test_common { + +auto ToIntList(const TypedValue &t) { + std::vector list; + for (auto x : t.ValueList()) { + list.push_back(x.ValueInt()); + } + return list; +}; + +auto ToIntMap(const TypedValue &t) { + std::map map; + for (const auto &kv : t.ValueMap()) map.emplace(kv.first, kv.second.ValueInt()); + return map; +}; + +std::string ToString(Expression *expr) { + std::ostringstream ss; + // PrintExpression(expr, &ss); + return ss.str(); +} + +std::string ToString(NamedExpression *expr) { + std::ostringstream ss; + // PrintExpression(expr, &ss); + return ss.str(); +} + +// Custom types for ORDER BY, SKIP, LIMIT, ON MATCH and ON CREATE expressions, +// so that they can be used to resolve function calls. +struct OrderBy { + std::vector expressions; +}; + +struct Skip { + Expression *expression = nullptr; +}; + +struct Limit { + Expression *expression = nullptr; +}; + +struct OnMatch { + std::vector set; +}; +struct OnCreate { + std::vector set; +}; + +// Helper functions for filling the OrderBy with expressions. +auto FillOrderBy(OrderBy &order_by, Expression *expression, Ordering ordering = Ordering::ASC) { + order_by.expressions.push_back({ordering, expression}); +} +template +auto FillOrderBy(OrderBy &order_by, Expression *expression, Ordering ordering, T... rest) { + FillOrderBy(order_by, expression, ordering); + FillOrderBy(order_by, rest...); +} +template +auto FillOrderBy(OrderBy &order_by, Expression *expression, T... rest) { + FillOrderBy(order_by, expression); + FillOrderBy(order_by, rest...); +} + +/// Create OrderBy expressions. +/// +/// The supported combination of arguments is: (Expression, [Ordering])+ +/// Since the Ordering is optional, by default it is ascending. +template +auto GetOrderBy(T... exprs) { + OrderBy order_by; + FillOrderBy(order_by, exprs...); + return order_by; +} + +/// Create PropertyLookup with given name and property. +/// +/// Name is used to create the Identifier which is used for property lookup. +template +auto GetPropertyLookup(AstStorage &storage, TDbAccessor &dba, const std::string &name, + memgraph::storage::v3::PropertyId property) { + return storage.Create(storage.Create(name), + storage.GetPropertyIx(dba.PropertyToName(property))); +} + +template +auto GetPropertyLookup(AstStorage &storage, TDbAccessor &dba, Expression *expr, + memgraph::storage::v3::PropertyId property) { + return storage.Create(expr, storage.GetPropertyIx(dba.PropertyToName(property))); +} + +template +auto GetPropertyLookup(AstStorage &storage, TDbAccessor &dba, Expression *expr, const std::string &property) { + return storage.Create(expr, storage.GetPropertyIx(property)); +} + +template +auto GetPropertyLookup(AstStorage &storage, TDbAccessor &, const std::string &name, + const std::pair &prop_pair) { + return storage.Create(storage.Create(name), storage.GetPropertyIx(prop_pair.first)); +} + +template +auto GetPropertyLookup(AstStorage &storage, TDbAccessor &, Expression *expr, + const std::pair &prop_pair) { + return storage.Create(expr, storage.GetPropertyIx(prop_pair.first)); +} + +/// Create an EdgeAtom with given name, direction and edge_type. +/// +/// Name is used to create the Identifier which is assigned to the edge. +auto GetEdge(AstStorage &storage, const std::string &name, EdgeAtom::Direction dir = EdgeAtom::Direction::BOTH, + const std::vector &edge_types = {}) { + std::vector types; + types.reserve(edge_types.size()); + for (const auto &type : edge_types) { + types.push_back(storage.GetEdgeTypeIx(type)); + } + return storage.Create(storage.Create(name), EdgeAtom::Type::SINGLE, dir, types); +} + +/// Create a variable length expansion EdgeAtom with given name, direction and +/// edge_type. +/// +/// Name is used to create the Identifier which is assigned to the edge. +auto GetEdgeVariable(AstStorage &storage, const std::string &name, EdgeAtom::Type type = EdgeAtom::Type::DEPTH_FIRST, + EdgeAtom::Direction dir = EdgeAtom::Direction::BOTH, + const std::vector &edge_types = {}, Identifier *flambda_inner_edge = nullptr, + Identifier *flambda_inner_node = nullptr, Identifier *wlambda_inner_edge = nullptr, + Identifier *wlambda_inner_node = nullptr, Expression *wlambda_expression = nullptr, + Identifier *total_weight = nullptr) { + std::vector types; + types.reserve(edge_types.size()); + for (const auto &type : edge_types) { + types.push_back(storage.GetEdgeTypeIx(type)); + } + auto r_val = storage.Create(storage.Create(name), type, dir, types); + + r_val->filter_lambda_.inner_edge = + flambda_inner_edge ? flambda_inner_edge : storage.Create(memgraph::utils::RandomString(20)); + r_val->filter_lambda_.inner_node = + flambda_inner_node ? flambda_inner_node : storage.Create(memgraph::utils::RandomString(20)); + + if (type == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) { + r_val->weight_lambda_.inner_edge = + wlambda_inner_edge ? wlambda_inner_edge : storage.Create(memgraph::utils::RandomString(20)); + r_val->weight_lambda_.inner_node = + wlambda_inner_node ? wlambda_inner_node : storage.Create(memgraph::utils::RandomString(20)); + r_val->weight_lambda_.expression = + wlambda_expression ? wlambda_expression : storage.Create(1); + + r_val->total_weight_ = total_weight; + } + + return r_val; +} + +/// Create a NodeAtom with given name and label. +/// +/// Name is used to create the Identifier which is assigned to the node. +auto GetNode(AstStorage &storage, const std::string &name, std::optional label = std::nullopt) { + auto node = storage.Create(storage.Create(name)); + if (label) node->labels_.emplace_back(storage.GetLabelIx(*label)); + return node; +} + +/// Create a Pattern with given atoms. +auto GetPattern(AstStorage &storage, std::vector atoms) { + auto pattern = storage.Create(); + pattern->identifier_ = storage.Create(memgraph::utils::RandomString(20), false); + pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); + return pattern; +} + +/// Create a Pattern with given name and atoms. +auto GetPattern(AstStorage &storage, const std::string &name, std::vector atoms) { + auto pattern = storage.Create(); + pattern->identifier_ = storage.Create(name, true); + pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); + return pattern; +} + +/// This function fills an AST node which with given patterns. +/// +/// The function is most commonly used to create Match and Create clauses. +template +auto GetWithPatterns(TWithPatterns *with_patterns, std::vector patterns) { + with_patterns->patterns_.insert(with_patterns->patterns_.begin(), patterns.begin(), patterns.end()); + return with_patterns; +} + +/// Create a query with given clauses. + +auto GetSingleQuery(SingleQuery *single_query, Clause *clause) { + single_query->clauses_.emplace_back(clause); + return single_query; +} +auto GetSingleQuery(SingleQuery *single_query, Match *match, Where *where) { + match->where_ = where; + single_query->clauses_.emplace_back(match); + return single_query; +} +auto GetSingleQuery(SingleQuery *single_query, With *with, Where *where) { + with->where_ = where; + single_query->clauses_.emplace_back(with); + return single_query; +} +template +auto GetSingleQuery(SingleQuery *single_query, Match *match, Where *where, T *...clauses) { + match->where_ = where; + single_query->clauses_.emplace_back(match); + return GetSingleQuery(single_query, clauses...); +} +template +auto GetSingleQuery(SingleQuery *single_query, With *with, Where *where, T *...clauses) { + with->where_ = where; + single_query->clauses_.emplace_back(with); + return GetSingleQuery(single_query, clauses...); +} + +template +auto GetSingleQuery(SingleQuery *single_query, Clause *clause, T *...clauses) { + single_query->clauses_.emplace_back(clause); + return GetSingleQuery(single_query, clauses...); +} + +auto GetCypherUnion(CypherUnion *cypher_union, SingleQuery *single_query) { + cypher_union->single_query_ = single_query; + return cypher_union; +} + +auto GetQuery(AstStorage &storage, SingleQuery *single_query) { + auto *query = storage.Create(); + query->single_query_ = single_query; + return query; +} + +template +auto GetQuery(AstStorage &storage, SingleQuery *single_query, T *...cypher_unions) { + auto *query = storage.Create(); + query->single_query_ = single_query; + query->cypher_unions_ = std::vector{cypher_unions...}; + return query; +} + +// Helper functions for constructing RETURN and WITH clauses. +void FillReturnBody(AstStorage &, ReturnBody &body, NamedExpression *named_expr) { + body.named_expressions.emplace_back(named_expr); +} +void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &name) { + if (name == "*") { + body.all_identifiers = true; + } else { + auto *ident = storage.Create(name); + auto *named_expr = storage.Create(name, ident); + body.named_expressions.emplace_back(named_expr); + } +} +void FillReturnBody(AstStorage &, ReturnBody &body, Limit limit) { body.limit = limit.expression; } +void FillReturnBody(AstStorage &, ReturnBody &body, Skip skip, Limit limit = Limit{}) { + body.skip = skip.expression; + body.limit = limit.expression; +} +void FillReturnBody(AstStorage &, ReturnBody &body, OrderBy order_by, Limit limit = Limit{}) { + body.order_by = order_by.expressions; + body.limit = limit.expression; +} +void FillReturnBody(AstStorage &, ReturnBody &body, OrderBy order_by, Skip skip, Limit limit = Limit{}) { + body.order_by = order_by.expressions; + body.skip = skip.expression; + body.limit = limit.expression; +} +void FillReturnBody(AstStorage &, ReturnBody &body, Expression *expr, NamedExpression *named_expr) { + // This overload supports `RETURN(expr, AS(name))` construct, since + // NamedExpression does not inherit Expression. + named_expr->expression_ = expr; + body.named_expressions.emplace_back(named_expr); +} +void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &name, NamedExpression *named_expr) { + named_expr->expression_ = storage.Create(name); + body.named_expressions.emplace_back(named_expr); +} +template +void FillReturnBody(AstStorage &storage, ReturnBody &body, Expression *expr, NamedExpression *named_expr, T... rest) { + named_expr->expression_ = expr; + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(AstStorage &storage, ReturnBody &body, NamedExpression *named_expr, T... rest) { + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &name, NamedExpression *named_expr, + T... rest) { + named_expr->expression_ = storage.Create(name); + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} +template +void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &name, T... rest) { + auto *ident = storage.Create(name); + auto *named_expr = storage.Create(name, ident); + body.named_expressions.emplace_back(named_expr); + FillReturnBody(storage, body, rest...); +} + +/// Create the return clause with given expressions. +/// +/// The supported expression combination of arguments is: +/// +/// (String | NamedExpression | (Expression NamedExpression))+ +/// [OrderBy] [Skip] [Limit] +/// +/// When the pair (Expression NamedExpression) is given, the Expression will be +/// moved inside the NamedExpression. This is done, so that the constructs like +/// RETURN(expr, AS("name"), ...) are supported. Taking a String is a shorthand +/// for RETURN(IDENT(string), AS(string), ....). +/// +/// @sa GetWith +template +auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { + auto ret = storage.Create(); + ret->body_.distinct = distinct; + FillReturnBody(storage, ret->body_, exprs...); + return ret; +} + +/// Create the with clause with given expressions. +/// +/// The supported expression combination is the same as for @c GetReturn. +/// +/// @sa GetReturn +template +auto GetWith(AstStorage &storage, bool distinct, T... exprs) { + auto with = storage.Create(); + with->body_.distinct = distinct; + FillReturnBody(storage, with->body_, exprs...); + return with; +} + +/// Create the UNWIND clause with given named expression. +auto GetUnwind(AstStorage &storage, NamedExpression *named_expr) { + return storage.Create(named_expr); +} +auto GetUnwind(AstStorage &storage, Expression *expr, NamedExpression *as) { + as->expression_ = expr; + return GetUnwind(storage, as); +} + +/// Create the delete clause with given named expressions. +auto GetDelete(AstStorage &storage, std::vector exprs, bool detach = false) { + auto del = storage.Create(); + del->expressions_.insert(del->expressions_.begin(), exprs.begin(), exprs.end()); + del->detach_ = detach; + return del; +} + +/// Create a set property clause for given property lookup and the right hand +/// side expression. +auto GetSet(AstStorage &storage, PropertyLookup *prop_lookup, Expression *expr) { + return storage.Create(prop_lookup, expr); +} + +/// Create a set properties clause for given identifier name and the right hand +/// side expression. +auto GetSet(AstStorage &storage, const std::string &name, Expression *expr, bool update = false) { + return storage.Create(storage.Create(name), expr, update); +} + +/// Create a set labels clause for given identifier name and labels. +auto GetSet(AstStorage &storage, const std::string &name, std::vector label_names) { + std::vector labels; + labels.reserve(label_names.size()); + for (const auto &label : label_names) { + labels.push_back(storage.GetLabelIx(label)); + } + return storage.Create(storage.Create(name), labels); +} + +/// Create a remove property clause for given property lookup +auto GetRemove(AstStorage &storage, PropertyLookup *prop_lookup) { return storage.Create(prop_lookup); } + +/// Create a remove labels clause for given identifier name and labels. +auto GetRemove(AstStorage &storage, const std::string &name, std::vector label_names) { + std::vector labels; + labels.reserve(label_names.size()); + for (const auto &label : label_names) { + labels.push_back(storage.GetLabelIx(label)); + } + return storage.Create(storage.Create(name), labels); +} + +/// Create a Merge clause for given Pattern with optional OnMatch and OnCreate +/// parts. +auto GetMerge(AstStorage &storage, Pattern *pattern, OnCreate on_create = OnCreate{}) { + auto *merge = storage.Create(); + merge->pattern_ = pattern; + merge->on_create_ = on_create.set; + return merge; +} +auto GetMerge(AstStorage &storage, Pattern *pattern, OnMatch on_match, OnCreate on_create = OnCreate{}) { + auto *merge = storage.Create(); + merge->pattern_ = pattern; + merge->on_match_ = on_match.set; + merge->on_create_ = on_create.set; + return merge; +} + +auto GetCallProcedure(AstStorage &storage, std::string procedure_name, + std::vector arguments = {}) { + auto *call_procedure = storage.Create(); + call_procedure->procedure_name_ = std::move(procedure_name); + call_procedure->arguments_ = std::move(arguments); + return call_procedure; +} + +/// Create the FOREACH clause with given named expression. +auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vector &clauses) { + return storage.Create(named_expr, clauses); +} + +} // namespace test_common + +} // namespace memgraph::query::v2 + +/// All the following macros implicitly pass `storage` variable to functions. +/// You need to have `AstStorage storage;` somewhere in scope to use them. +/// Refer to function documentation to see what the macro does. +/// +/// Example usage: +/// +/// // Create MATCH (n) -[r]- (m) RETURN m AS new_name +/// AstStorage storage; +/// auto query = QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), +/// RETURN(NEXPR("new_name"), IDENT("m"))); +#define NODE(...) memgraph::expr::test_common::GetNode(storage, __VA_ARGS__) +#define EDGE(...) memgraph::expr::test_common::GetEdge(storage, __VA_ARGS__) +#define EDGE_VARIABLE(...) memgraph::expr::test_common::GetEdgeVariable(storage, __VA_ARGS__) +#define PATTERN(...) memgraph::expr::test_common::GetPattern(storage, {__VA_ARGS__}) +#define PATTERN(...) memgraph::expr::test_common::GetPattern(storage, {__VA_ARGS__}) +#define NAMED_PATTERN(name, ...) memgraph::expr::test_common::GetPattern(storage, name, {__VA_ARGS__}) +#define OPTIONAL_MATCH(...) \ + memgraph::expr::test_common::GetWithPatterns(storage.Create(true), {__VA_ARGS__}) +#define MATCH(...) \ + memgraph::expr::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) +#define WHERE(expr) storage.Create((expr)) +#define CREATE(...) \ + memgraph::expr::test_common::GetWithPatterns(storage.Create(), {__VA_ARGS__}) +#define IDENT(...) storage.Create(__VA_ARGS__) +#define LITERAL(val) storage.Create((val)) +#define LIST(...) \ + storage.Create(std::vector{__VA_ARGS__}) +#define MAP(...) \ + storage.Create( \ + std::unordered_map{__VA_ARGS__}) +#define PROPERTY_PAIR(property_name) \ + std::make_pair(property_name, dba.NameToProperty(property_name)) // This one might not be needed at all +#define PRIMARY_PROPERTY_PAIR(property_name) std::make_pair(property_name, dba.NameToPrimaryProperty(property_name)) +#define SECONDARY_PROPERTY_PAIR(property_name) std::make_pair(property_name, dba.NameToSecondaryProperty(property_name)) +#define PROPERTY_LOOKUP(...) memgraph::expr::test_common::GetPropertyLookup(storage, dba, __VA_ARGS__) +#define PARAMETER_LOOKUP(token_position) storage.Create((token_position)) +#define NEXPR(name, expr) storage.Create((name), (expr)) +// AS is alternative to NEXPR which does not initialize NamedExpression with +// Expression. It should be used with RETURN or WITH. For example: +// RETURN(IDENT("n"), AS("n")) vs. RETURN(NEXPR("n", IDENT("n"))). +#define AS(name) storage.Create((name)) +#define RETURN(...) memgraph::expr::test_common::GetReturn(storage, false, __VA_ARGS__) +#define WITH(...) memgraph::expr::test_common::GetWith(storage, false, __VA_ARGS__) +#define RETURN_DISTINCT(...) memgraph::expr::test_common::GetReturn(storage, true, __VA_ARGS__) +#define WITH_DISTINCT(...) memgraph::expr::test_common::GetWith(storage, true, __VA_ARGS__) +#define UNWIND(...) memgraph::expr::test_common::GetUnwind(storage, __VA_ARGS__) +#define ORDER_BY(...) memgraph::expr::test_common::GetOrderBy(__VA_ARGS__) +#define SKIP(expr) \ + memgraph::expr::test_common::Skip { (expr) } +#define LIMIT(expr) \ + memgraph::expr::test_common::Limit { (expr) } +#define DELETE(...) memgraph::expr::test_common::GetDelete(storage, {__VA_ARGS__}) +#define DETACH_DELETE(...) memgraph::expr::test_common::GetDelete(storage, {__VA_ARGS__}, true) +#define SET(...) memgraph::expr::test_common::GetSet(storage, __VA_ARGS__) +#define REMOVE(...) memgraph::expr::test_common::GetRemove(storage, __VA_ARGS__) +#define MERGE(...) memgraph::expr::test_common::GetMerge(storage, __VA_ARGS__) +#define ON_MATCH(...) \ + memgraph::expr::test_common::OnMatch { \ + std::vector { __VA_ARGS__ } \ + } +#define ON_CREATE(...) \ + memgraph::expr::test_common::OnCreate { \ + std::vector { __VA_ARGS__ } \ + } +#define CREATE_INDEX_ON(label, property) \ + storage.Create(memgraph::query::v2::IndexQuery::Action::CREATE, (label), \ + std::vector{(property)}) +#define QUERY(...) memgraph::expr::test_common::GetQuery(storage, __VA_ARGS__) +#define SINGLE_QUERY(...) memgraph::expr::test_common::GetSingleQuery(storage.Create(), __VA_ARGS__) +#define UNION(...) memgraph::expr::test_common::GetCypherUnion(storage.Create(true), __VA_ARGS__) +#define UNION_ALL(...) memgraph::expr::test_common::GetCypherUnion(storage.Create(false), __VA_ARGS__) +#define FOREACH(...) memgraph::expr::test_common::GetForeach(storage, __VA_ARGS__) +// Various operators +#define NOT(expr) storage.Create((expr)) +#define UPLUS(expr) storage.Create((expr)) +#define UMINUS(expr) storage.Create((expr)) +#define IS_NULL(expr) storage.Create((expr)) +#define ADD(expr1, expr2) storage.Create((expr1), (expr2)) +#define LESS(expr1, expr2) storage.Create((expr1), (expr2)) +#define LESS_EQ(expr1, expr2) storage.Create((expr1), (expr2)) +#define GREATER(expr1, expr2) storage.Create((expr1), (expr2)) +#define GREATER_EQ(expr1, expr2) storage.Create((expr1), (expr2)) +#define SUM(expr) \ + storage.Create((expr), nullptr, memgraph::query::v2::Aggregation::Op::SUM) +#define COUNT(expr) \ + storage.Create((expr), nullptr, memgraph::query::v2::Aggregation::Op::COUNT) +#define AVG(expr) \ + storage.Create((expr), nullptr, memgraph::query::v2::Aggregation::Op::AVG) +#define COLLECT_LIST(expr) \ + storage.Create((expr), nullptr, memgraph::query::v2::Aggregation::Op::COLLECT_LIST) +#define EQ(expr1, expr2) storage.Create((expr1), (expr2)) +#define NEQ(expr1, expr2) storage.Create((expr1), (expr2)) +#define AND(expr1, expr2) storage.Create((expr1), (expr2)) +#define OR(expr1, expr2) storage.Create((expr1), (expr2)) +#define IN_LIST(expr1, expr2) storage.Create((expr1), (expr2)) +#define IF(cond, then, else) storage.Create((cond), (then), (else)) +// Function call +#define FN(function_name, ...) \ + storage.Create(memgraph::utils::ToUpperCase(function_name), \ + std::vector{__VA_ARGS__}) +// List slicing +#define SLICE(list, lower_bound, upper_bound) \ + storage.Create(list, lower_bound, upper_bound) +// all(variable IN list WHERE predicate) +#define ALL(variable, list, where) \ + storage.Create(storage.Create(variable), list, where) +#define SINGLE(variable, list, where) \ + storage.Create(storage.Create(variable), list, where) +#define ANY(variable, list, where) \ + storage.Create(storage.Create(variable), list, where) +#define NONE(variable, list, where) \ + storage.Create(storage.Create(variable), list, where) +#define REDUCE(accumulator, initializer, variable, list, expr) \ + storage.Create(storage.Create(accumulator), \ + initializer, storage.Create(variable), \ + list, expr) +#define COALESCE(...) \ + storage.Create(std::vector{__VA_ARGS__}) +#define EXTRACT(variable, list, expr) \ + storage.Create(storage.Create(variable), list, expr) +#define AUTH_QUERY(action, user, role, user_or_role, password, privileges) \ + storage.Create((action), (user), (role), (user_or_role), password, (privileges)) +#define DROP_USER(usernames) storage.Create((usernames)) +#define CALL_PROCEDURE(...) memgraph::query::v2::test_common::GetCallProcedure(storage, __VA_ARGS__) diff --git a/tests/unit/query_plan_v2.cpp b/tests/unit/query_v2_plan.cpp similarity index 95% rename from tests/unit/query_plan_v2.cpp rename to tests/unit/query_v2_plan.cpp index 009d0ca56..2eb8295db 100644 --- a/tests/unit/query_plan_v2.cpp +++ b/tests/unit/query_v2_plan.cpp @@ -22,14 +22,13 @@ #include #include -#include "query/v2/frontend/ast/ast.hpp" -// #include "query/frontend/semantic/symbol_generator.hpp" #include "expr/semantic/symbol_generator.hpp" #include "query/frontend/semantic/symbol_table.hpp" +#include "query/v2/frontend/ast/ast.hpp" #include "query/v2/plan/operator.hpp" #include "query/v2/plan/planner.hpp" -#include "query_common.hpp" +#include "query_v2_common.hpp" namespace memgraph::query { ::std::ostream &operator<<(::std::ostream &os, const Symbol &sym) { @@ -39,10 +38,10 @@ namespace memgraph::query { // using namespace memgraph::query::v2::plan; using namespace memgraph::expr::plan; -using memgraph::query::AstStorage; -using memgraph::query::SingleQuery; using memgraph::query::Symbol; using memgraph::query::SymbolGenerator; +using memgraph::query::v2::AstStorage; +using memgraph::query::v2::SingleQuery; using memgraph::query::v2::SymbolTable; using Type = memgraph::query::v2::EdgeAtom::Type; using Direction = memgraph::query::v2::EdgeAtom::Direction; @@ -75,8 +74,8 @@ auto CheckPlan(LogicalOperator &plan, const SymbolTable &symbol_table, TChecker. } template -auto CheckPlan(memgraph::query::CypherQuery *query, AstStorage &storage, TChecker... checker) { - auto symbol_table = memgraph::query::MakeSymbolTable(query); +auto CheckPlan(memgraph::query::v2::CypherQuery *query, AstStorage &storage, TChecker... checker) { + auto symbol_table = memgraph::expr::MakeSymbolTable(query); FakeDistributedDbAccessor dba; auto planner = MakePlanner(&dba, storage, symbol_table, query); CheckPlan(planner.plan(), symbol_table, checker...); @@ -95,45 +94,91 @@ void DeleteListContent(std::list *list) { TYPED_TEST_CASE(TestPlanner, PlannerTypes); TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { - FakeDistributedDbAccessor dba; - auto label = dba.Label("prim_label_one"); - auto prim_prop_one = PRIMARY_PROPERTY_PAIR("prim_prop_one"); - // auto prim_prop_two = PRIMARY_PROPERTY_PAIR("prim_prop_two"); - auto sec_prop_one = PRIMARY_PROPERTY_PAIR("sec_prop_one"); - auto sec_prop_two = PRIMARY_PROPERTY_PAIR("sec_prop_two"); - auto sec_prop_three = PRIMARY_PROPERTY_PAIR("sec_prop_three"); - dba.SetIndexCount(label, 1); - dba.SetIndexCount(label, prim_prop_one.second, 1); - // dba.SetIndexCount(label, prim_prop_two.second, 1); - dba.SetIndexCount(label, sec_prop_one.second, 1); - dba.SetIndexCount(label, sec_prop_two.second, 1); - dba.SetIndexCount(label, sec_prop_three.second, 1); - memgraph::query::v2::AstStorage storage; + const char *prim_label_name = "prim_label_one"; + // Exact primary key match, one elem as PK. { + FakeDistributedDbAccessor dba; + auto label = dba.Label(prim_label_name); + auto prim_prop_one = PRIMARY_PROPERTY_PAIR("prim_prop_one"); + + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, prim_prop_one.second, 1); + + dba.CreateSchema(label, {prim_prop_one.second}); + + memgraph::query::v2::AstStorage storage; + memgraph::query::v2::Expression *expected_primary_key; - - // Pray and hope for the best... expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); - - // auto asd1 = NODE("n", "label"); - // auto asd2 = PATTERN(NODE("n", "label")); - // auto asd3 = MATCH_V2(PATTERN(NODE("n", "label"))); - // auto asd4 = WHERE_V2(PROPERTY_LOOKUP("n", prim_prop_one)); - - auto *query = QUERY(SINGLE_QUERY_V2(MATCH_V2(PATTERN(NODE("n", "label"))), - WHERE_V2(PROPERTY_LOOKUP("n", prim_prop_one)), RETURN("n"))); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", prim_label_name))), + WHERE(EQ(PROPERTY_LOOKUP("n", prim_prop_one), LITERAL(1))), RETURN("n"))); auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); auto planner = MakePlanner(&dba, storage, symbol_table, query); CheckPlan(planner.plan(), symbol_table, ExpectScanAllByPrimaryKey(label, {expected_primary_key}), ExpectProduce()); + } + // Exact primary key match, two elem as PK. + { + FakeDistributedDbAccessor dba; + auto label = dba.Label(prim_label_name); + auto prim_prop_one = PRIMARY_PROPERTY_PAIR("prim_prop_one"); - // // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL RETURN n - // auto *query2 = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), - // WHERE(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop)))), RETURN("n"))); - // auto symbol_table = memgraph::query::MakeSymbolTable(query); - // auto planner = MakePlanner(&dba, storage, symbol_table, query); - // // We expect ScanAllByLabelProperty to come instead of ScanAll > Filter. - // CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectExpand(), - // ExpectProduce()); + auto prim_prop_two = PRIMARY_PROPERTY_PAIR("prim_prop_two"); + auto sec_prop_one = PRIMARY_PROPERTY_PAIR("sec_prop_one"); + auto sec_prop_two = PRIMARY_PROPERTY_PAIR("sec_prop_two"); + auto sec_prop_three = PRIMARY_PROPERTY_PAIR("sec_prop_three"); + + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, prim_prop_one.second, 1); + + dba.CreateSchema(label, {prim_prop_one.second, prim_prop_two.second}); + + dba.SetIndexCount(label, prim_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_one.second, 1); + dba.SetIndexCount(label, sec_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_three.second, 1); + memgraph::query::v2::AstStorage storage; + + memgraph::query::v2::Expression *expected_primary_key; + expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", prim_label_name))), + WHERE(AND(EQ(PROPERTY_LOOKUP("n", prim_prop_one), LITERAL(1)), + EQ(PROPERTY_LOOKUP("n", prim_prop_two), LITERAL(1)))), + RETURN("n"))); + auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByPrimaryKey(label, {expected_primary_key}), ExpectProduce()); + } + // One elem is missing from PK, default to ScanAllByLabelPropertyValue. + { + FakeDistributedDbAccessor dba; + auto label = dba.Label(prim_label_name); + + auto prim_prop_one = PRIMARY_PROPERTY_PAIR("prim_prop_one"); + auto prim_prop_two = PRIMARY_PROPERTY_PAIR("prim_prop_two"); + + auto sec_prop_one = PRIMARY_PROPERTY_PAIR("sec_prop_one"); + auto sec_prop_two = PRIMARY_PROPERTY_PAIR("sec_prop_two"); + auto sec_prop_three = PRIMARY_PROPERTY_PAIR("sec_prop_three"); + + dba.SetIndexCount(label, 1); + dba.SetIndexCount(label, prim_prop_one.second, 1); + + dba.CreateSchema(label, {prim_prop_one.second, prim_prop_two.second}); + + dba.SetIndexCount(label, prim_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_one.second, 1); + dba.SetIndexCount(label, sec_prop_two.second, 1); + dba.SetIndexCount(label, sec_prop_three.second, 1); + memgraph::query::v2::AstStorage storage; + + memgraph::query::v2::Expression *expected_primary_key; + expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); + auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", prim_label_name))), + WHERE(EQ(PROPERTY_LOOKUP("n", prim_prop_one), LITERAL(1))), RETURN("n"))); + auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); + auto planner = MakePlanner(&dba, storage, symbol_table, query); + CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, prim_prop_one, IDENT("n")), + ExpectProduce()); } } From 59f4b893612f187d9abe7f0bb0cf02b07d0dcb79 Mon Sep 17 00:00:00 2001 From: jbajic Date: Tue, 6 Dec 2022 12:34:36 +0100 Subject: [PATCH 006/251] Remove redundandt shard properties --- src/storage/v3/shard.cpp | 5 +---- src/storage/v3/shard.hpp | 32 -------------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index a77f42a76..39c0683cd 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -332,10 +332,7 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key, vertex_validator_{schema_validator_, primary_label}, indices_{config.items, vertex_validator_}, isolation_level_{config.transaction.isolation_level}, - config_{config}, - uuid_{utils::GenerateUUID()}, - epoch_id_{utils::GenerateUUID()}, - global_locker_{file_retainer_.AddLocker()} { + config_{config} { CreateSchema(primary_label_, schema); StoreMapping(std::move(id_to_name)); } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index b998c06cc..31259a0fe 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -393,38 +393,6 @@ class Shard final { // storage. std::list deleted_edges_; - // UUID used to distinguish snapshots and to link snapshots to WALs - std::string uuid_; - // Sequence number used to keep track of the chain of WALs. - uint64_t wal_seq_num_{0}; - - // UUID to distinguish different main instance runs for replication process - // on SAME storage. - // Multiple instances can have same storage UUID and be MAIN at the same time. - // We cannot compare commit timestamps of those instances if one of them - // becomes the replica of the other so we use epoch_id_ as additional - // discriminating property. - // Example of this: - // We have 2 instances of the same storage, S1 and S2. - // S1 and S2 are MAIN and accept their own commits and write them to the WAL. - // At the moment when S1 commited a transaction with timestamp 20, and S2 - // a different transaction with timestamp 15, we change S2's role to REPLICA - // and register it on S1. - // Without using the epoch_id, we don't know that S1 and S2 have completely - // different transactions, we think that the S2 is behind only by 5 commits. - std::string epoch_id_; - // History of the previous epoch ids. - // Each value consists of the epoch id along the last commit belonging to that - // epoch. - std::deque> epoch_history_; - - uint64_t wal_unsynced_transactions_{0}; - - utils::FileRetainer file_retainer_; - - // Global locker that is used for clients file locking - utils::FileRetainer::FileLocker global_locker_; - // Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but // haven't been cleaned up yet std::map> start_logical_id_to_transaction_{}; From 153e9e2fac0a8b628e32585b1dd524f95811420f Mon Sep 17 00:00:00 2001 From: jbajic Date: Wed, 7 Dec 2022 14:13:14 +0100 Subject: [PATCH 007/251] Begin split funcionlity from shard side --- src/storage/v3/shard.cpp | 39 ++++++++++++++++++++++++++++++--- src/storage/v3/shard.hpp | 18 +++++++++++++++ src/storage/v3/shard_rsm.hpp | 3 +++ src/storage/v3/shard_worker.hpp | 8 +++++++ 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 39c0683cd..1b3efbff4 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -18,10 +18,7 @@ #include #include #include -#include -#include -#include #include #include "io/network/endpoint.hpp" @@ -1042,6 +1039,42 @@ void Shard::StoreMapping(std::unordered_map id_to_name) { name_id_mapper_.StoreMapping(std::move(id_to_name)); } +std::optional Shard::ShouldSplit() const noexcept { + if (vertices_.size() > 10000000) { + // Why should we care if the selected vertex is deleted + auto mid_elem = vertices_.begin(); + // mid_elem->first + std::ranges::advance(mid_elem, static_cast(vertices_.size() / 2)); + return SplitInfo{shard_version_, mid_elem->first}; + } + return std::nullopt; +} + +SplitData Shard::PerformSplit(const PrimaryKey &split_key) { + SplitData data; + data.vertices = std::map(vertices_.find(split_key), vertices_.end()); + data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; + + // Get all edges related with those vertices + if (config_.items.properties_on_edges) { + data.edges = std::invoke([&split_vertices = data.vertices]() { + // How to reserve? + EdgeContainer split_edges; + for (const auto &vertex : split_vertices) { + for (const auto &in_edge : vertex.second.in_edges) { + auto edge = std::get<2>(in_edge).ptr; + split_edges.insert(edge->gid, Edge{.gid = edge->gid, .delta = edge->delta, .properties = edge->properties}); + } + } + return split_edges; + }); + } + // TODO We also need to send ongoing transactions to the shard + // since they own deltas + + return data; +} + bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const { return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ && (!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_); diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 31259a0fe..301fc8132 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -175,6 +175,19 @@ struct SchemasInfo { Schemas::SchemasList schemas; }; +struct SplitInfo { + uint64_t shard_version; + PrimaryKey split_point; +}; + +// If edge properties-on-edges is false then we don't need to send edges but +// only vertices, since they will contain those edges +struct SplitData { + VertexContainer vertices; + std::optional edges; + IndicesInfo indices_info; +}; + /// Structure used to return information about the storage. struct StorageInfo { uint64_t vertex_count; @@ -357,6 +370,10 @@ class Shard final { void StoreMapping(std::unordered_map id_to_name); + std::optional ShouldSplit() const noexcept; + + SplitData PerformSplit(const PrimaryKey &split_key); + private: Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level); @@ -374,6 +391,7 @@ class Shard final { // list is used only when properties are enabled for edges. Because of that we // keep a separate count of edges that is always updated. uint64_t edge_count_{0}; + uint64_t shard_version_{0}; SchemaValidator schema_validator_; VertexValidator vertex_validator_; diff --git a/src/storage/v3/shard_rsm.hpp b/src/storage/v3/shard_rsm.hpp index d301bf40b..ba284a3ca 100644 --- a/src/storage/v3/shard_rsm.hpp +++ b/src/storage/v3/shard_rsm.hpp @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include @@ -41,6 +42,8 @@ class ShardRsm { public: explicit ShardRsm(std::unique_ptr &&shard) : shard_(std::move(shard)){}; + std::optional ShouldSplit() const noexcept { return shard_->ShouldSplit(); } + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) msgs::ReadResponses Read(msgs::ReadRequests requests) { return std::visit([&](auto &&request) mutable { return HandleRead(std::forward(request)); }, diff --git a/src/storage/v3/shard_worker.hpp b/src/storage/v3/shard_worker.hpp index 547aa0a6f..dcdc6ee13 100644 --- a/src/storage/v3/shard_worker.hpp +++ b/src/storage/v3/shard_worker.hpp @@ -173,6 +173,14 @@ class ShardWorker { auto &rsm = rsm_map_.at(uuid); Time next_for_uuid = rsm.Cron(); + // Check if shard should split + if (const auto split_info = rsm.ShouldSplit(); split_info) { + // Request split from coordinator + // split_point => middle pk + // shard_id => uuid + // shard_version => + } + cron_schedule_.pop(); cron_schedule_.push(std::make_pair(next_for_uuid, uuid)); } else { From 4ed20f0247bc3510eaeca16d97df80423bbca7ba Mon Sep 17 00:00:00 2001 From: Kostas Kyrimis Date: Thu, 8 Dec 2022 18:46:30 +0200 Subject: [PATCH 008/251] Add prototype for CreateNode multiframe --- src/query/v2/plan/operator.cpp | 70 +++++++++++++++ tests/unit/CMakeLists.txt | 3 + .../unit/query_v2_create_node_multiframe.cpp | 87 +++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 tests/unit/query_v2_create_node_multiframe.cpp diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index c909cc466..da0913376 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -189,6 +189,17 @@ class DistributedCreateNodeCursor : public Cursor { return false; } + void PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("CreateNodeMF"); + input_cursor_->PullMultiple(multi_frame, context); + auto &request_router = context.request_router; + { + SCOPED_REQUEST_WAIT_PROFILE; + request_router->CreateVertices(NodeCreationInfoToRequests(context, multi_frame)); + } + PlaceNodesOnTheMultiFrame(multi_frame, context); + } + void Shutdown() override { input_cursor_->Shutdown(); } void Reset() override {} @@ -247,6 +258,65 @@ class DistributedCreateNodeCursor : public Cursor { return requests; } + void PlaceNodesOnTheMultiFrame(MultiFrame &multi_frame, ExecutionContext &context) { + auto multi_frame_reader = multi_frame.GetValidFramesConsumer(); + size_t i = 0; + MG_ASSERT(std::distance(multi_frame_reader.begin(), multi_frame_reader.end())); + for (auto &frame : multi_frame_reader) { + const auto primary_label = msgs::Label{.id = nodes_info_[0]->labels[0]}; + msgs::Vertex v{.id = std::make_pair(primary_label, primary_keys_[i])}; + frame[nodes_info_.front()->symbol] = TypedValue( + query::v2::accessors::VertexAccessor(std::move(v), src_vertex_props_[i++], context.request_router)); + } + } + + std::vector NodeCreationInfoToRequests(ExecutionContext &context, MultiFrame &multi_frame) { + std::vector requests; + auto multi_frame_reader = multi_frame.GetValidFramesConsumer(); + for (auto &frame : multi_frame_reader) { + msgs::PrimaryKey pk; + for (const auto &node_info : nodes_info_) { + msgs::NewVertex rqst; + MG_ASSERT(!node_info->labels.empty(), "Cannot determine primary label"); + const auto primary_label = node_info->labels[0]; + // TODO(jbajic) Fix properties not send, + // suggestion: ignore distinction between properties and primary keys + // since schema validation is done on storage side + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, + storage::v3::View::NEW); + if (const auto *node_info_properties = std::get_if(&node_info->properties)) { + for (const auto &[key, value_expression] : *node_info_properties) { + TypedValue val = value_expression->Accept(evaluator); + if (context.request_router->IsPrimaryKey(primary_label, key)) { + rqst.primary_key.push_back(TypedValueToValue(val)); + pk.push_back(TypedValueToValue(val)); + } + } + } else { + auto property_map = evaluator.Visit(*std::get(node_info->properties)).ValueMap(); + for (const auto &[key, value] : property_map) { + auto key_str = std::string(key); + auto property_id = context.request_router->NameToProperty(key_str); + if (context.request_router->IsPrimaryKey(primary_label, property_id)) { + rqst.primary_key.push_back(TypedValueToValue(value)); + pk.push_back(TypedValueToValue(value)); + } + } + } + + if (node_info->labels.empty()) { + throw QueryRuntimeException("Primary label must be defined!"); + } + // TODO(kostasrim) Copy non primary labels as well + rqst.label_ids.push_back(msgs::Label{.id = primary_label}); + src_vertex_props_.push_back(rqst.properties); + requests.push_back(std::move(rqst)); + } + primary_keys_.push_back(std::move(pk)); + } + return requests; + } + private: const UniqueCursorPtr input_cursor_; std::vector nodes_info_; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index b1c5c9c6f..731c9ea1e 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -414,3 +414,6 @@ target_link_libraries(${test_prefix}query_v2_expression_evaluator mg-query-v2) # Tests for multiframes add_unit_test(query_v2_create_expand_multiframe.cpp) target_link_libraries(${test_prefix}query_v2_create_expand_multiframe mg-query-v2) + +add_unit_test(query_v2_create_node_multiframe.cpp) +target_link_libraries(${test_prefix}query_v2_create_node_multiframe mg-query-v2) diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp new file mode 100644 index 000000000..b4a6bdbaf --- /dev/null +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -0,0 +1,87 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "mock_helpers.hpp" + +#include "query/v2/bindings/frame.hpp" +#include "query/v2/bindings/symbol_table.hpp" +#include "query/v2/common.hpp" +#include "query/v2/context.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/requests.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/shard.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" + +using namespace memgraph::query::v2; +using namespace memgraph::query::v2::plan; +namespace memgraph { +class TestTemplate : public testing::Test { + protected: + void SetUp() override {} +}; + +ExecutionContext MakeContext(const AstStorage &storage, const SymbolTable &symbol_table, RequestRouterInterface *router, + IdAllocator *id_alloc) { + ExecutionContext context; + context.symbol_table = symbol_table; + context.evaluation_context.properties = NamesToProperties(storage.properties_, router); + context.evaluation_context.labels = NamesToLabels(storage.labels_, router); + context.edge_ids_alloc = id_alloc; + context.request_router = router; + return context; +} + +MultiFrame CreateMultiFrame(const size_t max_pos) { + static constexpr size_t frame_size = 100; + MultiFrame multi_frame(max_pos, frame_size, utils::NewDeleteResource()); + auto frames_populator = multi_frame.GetInvalidFramesPopulator(); + for (auto &frame : frames_populator) { + frame.MakeValid(); + } + + return multi_frame; +} + +TEST_F(TestTemplate, CreateNode) { + MockedRequestRouter router; + + AstStorage ast; + SymbolTable symbol_table; + + query::v2::plan::NodeCreationInfo node; + query::v2::plan::EdgeCreationInfo edge; + edge.edge_type = msgs::EdgeTypeId::FromUint(1); + edge.direction = EdgeAtom::Direction::IN; + auto id_alloc = IdAllocator(0, 100); + + node.symbol = symbol_table.CreateSymbol("n", true); + node.labels.push_back(msgs::LabelId::FromUint(2)); + auto literal = query::v2::PrimitiveLiteral(); + literal.value_ = TypedValue(static_cast(200)); + auto p = query::v2::plan::PropertiesMapList{}; + p.push_back(std::make_pair(msgs::PropertyId::FromUint(2), &literal)); + node.properties.emplace<0>(std::move(p)); + + auto create_expand = query::v2::plan::CreateNode(nullptr, node); + auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); + + EXPECT_CALL(router, CreateVertices(testing::_)) + .Times(1) + .WillOnce(::testing::Return(std::vector{})); + EXPECT_CALL(router, IsPrimaryKey(testing::_, testing::_)).WillRepeatedly(::testing::Return(true)); + auto context = MakeContext(ast, symbol_table, &router, &id_alloc); + auto multi_frame = CreateMultiFrame(context.symbol_table.max_position()); + cursor->PullMultiple(multi_frame, context); +} +} // namespace memgraph From b968748d9f0932d9c4c2846c91b4dbb8afe5f538 Mon Sep 17 00:00:00 2001 From: jbajic Date: Tue, 6 Dec 2022 12:34:36 +0100 Subject: [PATCH 009/251] Remove redundandt shard properties --- src/storage/v3/shard.cpp | 5 +---- src/storage/v3/shard.hpp | 32 -------------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 0d8b21402..de5161572 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -332,10 +332,7 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key, vertex_validator_{schema_validator_, primary_label}, indices_{config.items, vertex_validator_}, isolation_level_{config.transaction.isolation_level}, - config_{config}, - uuid_{utils::GenerateUUID()}, - epoch_id_{utils::GenerateUUID()}, - global_locker_{file_retainer_.AddLocker()} { + config_{config} { CreateSchema(primary_label_, schema); StoreMapping(std::move(id_to_name)); } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 8542d47eb..ed170fd55 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -392,38 +392,6 @@ class Shard final { // storage. std::list deleted_edges_; - // UUID used to distinguish snapshots and to link snapshots to WALs - std::string uuid_; - // Sequence number used to keep track of the chain of WALs. - uint64_t wal_seq_num_{0}; - - // UUID to distinguish different main instance runs for replication process - // on SAME storage. - // Multiple instances can have same storage UUID and be MAIN at the same time. - // We cannot compare commit timestamps of those instances if one of them - // becomes the replica of the other so we use epoch_id_ as additional - // discriminating property. - // Example of this: - // We have 2 instances of the same storage, S1 and S2. - // S1 and S2 are MAIN and accept their own commits and write them to the WAL. - // At the moment when S1 commited a transaction with timestamp 20, and S2 - // a different transaction with timestamp 15, we change S2's role to REPLICA - // and register it on S1. - // Without using the epoch_id, we don't know that S1 and S2 have completely - // different transactions, we think that the S2 is behind only by 5 commits. - std::string epoch_id_; - // History of the previous epoch ids. - // Each value consists of the epoch id along the last commit belonging to that - // epoch. - std::deque> epoch_history_; - - uint64_t wal_unsynced_transactions_{0}; - - utils::FileRetainer file_retainer_; - - // Global locker that is used for clients file locking - utils::FileRetainer::FileLocker global_locker_; - // Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but // haven't been cleaned up yet std::map> start_logical_id_to_transaction_{}; From 486da0bd1cf6711f626145eff9334d56fc405525 Mon Sep 17 00:00:00 2001 From: jbajic Date: Wed, 7 Dec 2022 14:13:14 +0100 Subject: [PATCH 010/251] Begin split funcionlity from shard side --- src/storage/v3/shard.cpp | 39 ++++++++++++++++++++++++++++++--- src/storage/v3/shard.hpp | 18 +++++++++++++++ src/storage/v3/shard_rsm.hpp | 3 +++ src/storage/v3/shard_worker.hpp | 8 +++++++ 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index de5161572..294c99f65 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -18,10 +18,7 @@ #include #include #include -#include -#include -#include #include #include "io/network/endpoint.hpp" @@ -1045,6 +1042,42 @@ void Shard::StoreMapping(std::unordered_map id_to_name) { name_id_mapper_.StoreMapping(std::move(id_to_name)); } +std::optional Shard::ShouldSplit() const noexcept { + if (vertices_.size() > 10000000) { + // Why should we care if the selected vertex is deleted + auto mid_elem = vertices_.begin(); + // mid_elem->first + std::ranges::advance(mid_elem, static_cast(vertices_.size() / 2)); + return SplitInfo{shard_version_, mid_elem->first}; + } + return std::nullopt; +} + +SplitData Shard::PerformSplit(const PrimaryKey &split_key) { + SplitData data; + data.vertices = std::map(vertices_.find(split_key), vertices_.end()); + data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; + + // Get all edges related with those vertices + if (config_.items.properties_on_edges) { + data.edges = std::invoke([&split_vertices = data.vertices]() { + // How to reserve? + EdgeContainer split_edges; + for (const auto &vertex : split_vertices) { + for (const auto &in_edge : vertex.second.in_edges) { + auto edge = std::get<2>(in_edge).ptr; + split_edges.insert(edge->gid, Edge{.gid = edge->gid, .delta = edge->delta, .properties = edge->properties}); + } + } + return split_edges; + }); + } + // TODO We also need to send ongoing transactions to the shard + // since they own deltas + + return data; +} + bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const { return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ && (!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_); diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index ed170fd55..88849184d 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -174,6 +174,19 @@ struct SchemasInfo { Schemas::SchemasList schemas; }; +struct SplitInfo { + uint64_t shard_version; + PrimaryKey split_point; +}; + +// If edge properties-on-edges is false then we don't need to send edges but +// only vertices, since they will contain those edges +struct SplitData { + VertexContainer vertices; + std::optional edges; + IndicesInfo indices_info; +}; + /// Structure used to return information about the storage. struct StorageInfo { uint64_t vertex_count; @@ -356,6 +369,10 @@ class Shard final { void StoreMapping(std::unordered_map id_to_name); + std::optional ShouldSplit() const noexcept; + + SplitData PerformSplit(const PrimaryKey &split_key); + private: Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level); @@ -373,6 +390,7 @@ class Shard final { // list is used only when properties are enabled for edges. Because of that we // keep a separate count of edges that is always updated. uint64_t edge_count_{0}; + uint64_t shard_version_{0}; SchemaValidator schema_validator_; VertexValidator vertex_validator_; diff --git a/src/storage/v3/shard_rsm.hpp b/src/storage/v3/shard_rsm.hpp index d301bf40b..ba284a3ca 100644 --- a/src/storage/v3/shard_rsm.hpp +++ b/src/storage/v3/shard_rsm.hpp @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include @@ -41,6 +42,8 @@ class ShardRsm { public: explicit ShardRsm(std::unique_ptr &&shard) : shard_(std::move(shard)){}; + std::optional ShouldSplit() const noexcept { return shard_->ShouldSplit(); } + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) msgs::ReadResponses Read(msgs::ReadRequests requests) { return std::visit([&](auto &&request) mutable { return HandleRead(std::forward(request)); }, diff --git a/src/storage/v3/shard_worker.hpp b/src/storage/v3/shard_worker.hpp index 547aa0a6f..dcdc6ee13 100644 --- a/src/storage/v3/shard_worker.hpp +++ b/src/storage/v3/shard_worker.hpp @@ -173,6 +173,14 @@ class ShardWorker { auto &rsm = rsm_map_.at(uuid); Time next_for_uuid = rsm.Cron(); + // Check if shard should split + if (const auto split_info = rsm.ShouldSplit(); split_info) { + // Request split from coordinator + // split_point => middle pk + // shard_id => uuid + // shard_version => + } + cron_schedule_.pop(); cron_schedule_.push(std::make_pair(next_for_uuid, uuid)); } else { From 04450dada7f7bc1c301678044d12edb4925161f4 Mon Sep 17 00:00:00 2001 From: Kostas Kyrimis Date: Mon, 12 Dec 2022 19:23:40 +0200 Subject: [PATCH 011/251] Simplify tests --- .../unit/query_v2_create_node_multiframe.cpp | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp index b4a6bdbaf..f19082783 100644 --- a/tests/unit/query_v2_create_node_multiframe.cpp +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -20,28 +20,9 @@ #include "query/v2/requests.hpp" #include "storage/v3/property_value.hpp" #include "storage/v3/shard.hpp" -#include "utils/logging.hpp" #include "utils/memory.hpp" -using namespace memgraph::query::v2; -using namespace memgraph::query::v2::plan; -namespace memgraph { -class TestTemplate : public testing::Test { - protected: - void SetUp() override {} -}; - -ExecutionContext MakeContext(const AstStorage &storage, const SymbolTable &symbol_table, RequestRouterInterface *router, - IdAllocator *id_alloc) { - ExecutionContext context; - context.symbol_table = symbol_table; - context.evaluation_context.properties = NamesToProperties(storage.properties_, router); - context.evaluation_context.labels = NamesToLabels(storage.labels_, router); - context.edge_ids_alloc = id_alloc; - context.request_router = router; - return context; -} - +namespace memgraph::query::v2 { MultiFrame CreateMultiFrame(const size_t max_pos) { static constexpr size_t frame_size = 100; MultiFrame multi_frame(max_pos, frame_size, utils::NewDeleteResource()); @@ -53,29 +34,36 @@ MultiFrame CreateMultiFrame(const size_t max_pos) { return multi_frame; } -TEST_F(TestTemplate, CreateNode) { - MockedRequestRouter router; +TEST(CreateNodeTest, CreateNodeCursor) { + using testing::_; + using testing::Return; AstStorage ast; SymbolTable symbol_table; - query::v2::plan::NodeCreationInfo node; - query::v2::plan::EdgeCreationInfo edge; + plan::NodeCreationInfo node; + plan::EdgeCreationInfo edge; edge.edge_type = msgs::EdgeTypeId::FromUint(1); edge.direction = EdgeAtom::Direction::IN; auto id_alloc = IdAllocator(0, 100); node.symbol = symbol_table.CreateSymbol("n", true); node.labels.push_back(msgs::LabelId::FromUint(2)); - auto literal = query::v2::PrimitiveLiteral(); + auto literal = PrimitiveLiteral(); literal.value_ = TypedValue(static_cast(200)); - auto p = query::v2::plan::PropertiesMapList{}; + auto p = plan::PropertiesMapList{}; p.push_back(std::make_pair(msgs::PropertyId::FromUint(2), &literal)); node.properties.emplace<0>(std::move(p)); - auto create_expand = query::v2::plan::CreateNode(nullptr, node); - auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); + auto once_cur = plan::MakeUniqueCursorPtr(utils::NewDeleteResource()); + EXPECT_CALL(BaseToMock(once_cur.get()), PullMultiple(_, _)).Times(1); + std::shared_ptr once_op = std::make_shared(); + EXPECT_CALL(BaseToMock(once_op.get()), MakeCursor(_)).Times(1).WillOnce(Return(std::move(once_cur))); + + auto create_expand = plan::CreateNode(once_op, node); + auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); + MockedRequestRouter router; EXPECT_CALL(router, CreateVertices(testing::_)) .Times(1) .WillOnce(::testing::Return(std::vector{})); @@ -84,4 +72,4 @@ TEST_F(TestTemplate, CreateNode) { auto multi_frame = CreateMultiFrame(context.symbol_table.max_position()); cursor->PullMultiple(multi_frame, context); } -} // namespace memgraph +} // namespace memgraph::query::v2 From 3604046f686fa112f8bc8320078468f5fca51cf3 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 12 Dec 2022 10:53:07 +0100 Subject: [PATCH 012/251] Implement cypher query based simulation testing Make the Interpreter be able to handle SimulatorTransport as well. This includes introducing changes that make it possible to use the different transport types in a semi-polymorphic way with the introduction of factory methods in the RequestRouter. The reason for this solution is that the classes that represent the different transport types have member function templates, that we can not make virtual. This solution seemed to be the least convoluted. In the testing itself now it is possible to pass a set of cypher queried to the interpreter which would run these queries against the interpreter and the individual shards that are managed and started up by the MachineManager with the different entities communicating over the simulated network. --- src/io/local_transport/local_system.hpp | 2 + src/io/simulator/simulator.hpp | 4 +- src/io/simulator/simulator_transport.hpp | 2 +- src/memgraph.cpp | 14 ++- src/query/v2/interpreter.cpp | 32 ++--- src/query/v2/interpreter.hpp | 19 +-- src/query/v2/request_router.hpp | 109 ++++++++++++++++++ tests/simulation/CMakeLists.txt | 3 +- tests/simulation/cluster_property_test_v2.cpp | 65 +++++++++++ tests/simulation/simulation_interpreter.hpp | 95 +++++++++++++++ tests/simulation/test_cluster.hpp | 62 ++++++++++ 11 files changed, 368 insertions(+), 39 deletions(-) create mode 100644 tests/simulation/cluster_property_test_v2.cpp create mode 100644 tests/simulation/simulation_interpreter.hpp diff --git a/src/io/local_transport/local_system.hpp b/src/io/local_transport/local_system.hpp index 2e54f8d75..ec148b50b 100644 --- a/src/io/local_transport/local_system.hpp +++ b/src/io/local_transport/local_system.hpp @@ -29,6 +29,8 @@ class LocalSystem { return Io{local_transport, address}; } + std::shared_ptr &GetTransportHandle() { return local_transport_handle_; } + void ShutDown() { local_transport_handle_->ShutDown(); } }; diff --git a/src/io/simulator/simulator.hpp b/src/io/simulator/simulator.hpp index 622c264b4..5095ab58b 100644 --- a/src/io/simulator/simulator.hpp +++ b/src/io/simulator/simulator.hpp @@ -41,7 +41,7 @@ class Simulator { Io Register(Address address) { std::uniform_int_distribution seed_distrib; uint64_t seed = seed_distrib(rng_); - return Io{SimulatorTransport{simulator_handle_, address, seed}, address}; + return Io{SimulatorTransport(simulator_handle_, address, seed), address}; } void IncrementServerCountAndWaitForQuiescentState(Address address) { @@ -49,5 +49,7 @@ class Simulator { } SimulatorStats Stats() { return simulator_handle_->Stats(); } + + std::shared_ptr GetSimulatorHandle() { return simulator_handle_; } }; }; // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_transport.hpp b/src/io/simulator/simulator_transport.hpp index 5e5a24aa9..492b59d3a 100644 --- a/src/io/simulator/simulator_transport.hpp +++ b/src/io/simulator/simulator_transport.hpp @@ -25,7 +25,7 @@ using memgraph::io::Time; class SimulatorTransport { std::shared_ptr simulator_handle_; - const Address address_; + Address address_; std::mt19937 rng_; public: diff --git a/src/memgraph.cpp b/src/memgraph.cpp index d825cc0e7..ca03024fe 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -607,15 +607,15 @@ int main(int argc, char **argv) { // to minimize the impact of their failure on the main storage. memgraph::io::local_transport::LocalSystem ls; - auto unique_local_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); - auto io = ls.Register(unique_local_addr_query); + auto unique_local_coord_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); + auto io = ls.Register(unique_local_coord_addr_query); memgraph::machine_manager::MachineConfig config{ - .coordinator_addresses = std::vector{unique_local_addr_query}, + .coordinator_addresses = std::vector{unique_local_coord_addr_query}, .is_storage = true, .is_coordinator = true, - .listen_ip = unique_local_addr_query.last_known_ip, - .listen_port = unique_local_addr_query.last_known_port, + .listen_ip = unique_local_coord_addr_query.last_known_ip, + .listen_port = unique_local_coord_addr_query.last_known_port, }; memgraph::coordinator::ShardMap sm; @@ -640,6 +640,8 @@ int main(int argc, char **argv) { memgraph::machine_manager::MachineManager mm{io, config, coordinator}; std::jthread mm_thread([&mm] { mm.Run(); }); + auto rr_factory = std::make_unique(ls.GetTransportHandle()); + memgraph::query::v2::InterpreterContext interpreter_context{ (memgraph::storage::v3::Shard *)(nullptr), {.query = {.allow_load_csv = FLAGS_allow_load_csv}, @@ -650,7 +652,7 @@ int main(int argc, char **argv) { .stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries, .stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)}, FLAGS_data_directory, - std::move(io), + std::move(rr_factory), mm.CoordinatorAddress()}; SessionData session_data{&interpreter_context}; diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index 1c2d6dadf..c428d1a39 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -793,34 +793,24 @@ using RWType = plan::ReadWriteTypeChecker::RWType; InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config, const std::filesystem::path & /*data_directory*/, - io::Io io, + std::unique_ptr &&request_router_factory, coordinator::Address coordinator_addr) - : db(db), config(config), io{std::move(io)}, coordinator_address{coordinator_addr} {} + : db(db), + config(config), + coordinator_address{coordinator_addr}, + request_router_factory_{std::move(request_router_factory)} {} Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) { MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL"); - // TODO(tyler) make this deterministic so that it can be tested. - auto random_uuid = boost::uuids::uuid{boost::uuids::random_generator()()}; - auto query_io = interpreter_context_->io.ForkLocal(random_uuid); + request_router_ = + interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address); - request_router_ = std::make_unique>( - coordinator::CoordinatorClient( - query_io, interpreter_context_->coordinator_address, std::vector{interpreter_context_->coordinator_address}), - std::move(query_io)); // Get edge ids - coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}}; - io::rsm::WriteRequest ww; - ww.operation = requests; - auto resp = interpreter_context_->io - .Request, - io::rsm::WriteResponse>( - interpreter_context_->coordinator_address, ww) - .Wait(); - if (resp.HasValue()) { - const auto alloc_edge_id_reps = - std::get(resp.GetValue().message.write_return); - interpreter_context_->edge_ids_alloc = {alloc_edge_id_reps.low, alloc_edge_id_reps.high}; + const auto edge_ids_alloc_min_max_pair = + request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address); + if (edge_ids_alloc_min_max_pair) { + interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second}; } } diff --git a/src/query/v2/interpreter.hpp b/src/query/v2/interpreter.hpp index 985c9a90c..a83a26f11 100644 --- a/src/query/v2/interpreter.hpp +++ b/src/query/v2/interpreter.hpp @@ -17,6 +17,7 @@ #include "coordinator/coordinator.hpp" #include "coordinator/coordinator_client.hpp" #include "io/local_transport/local_transport.hpp" +#include "io/simulator/simulator_transport.hpp" #include "io/transport.hpp" #include "query/v2/auth_checker.hpp" #include "query/v2/bindings/cypher_main_visitor.hpp" @@ -172,7 +173,8 @@ struct PreparedQuery { struct InterpreterContext { explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config, const std::filesystem::path &data_directory, - io::Io io, coordinator::Address coordinator_addr); + std::unique_ptr &&request_router_factory, + coordinator::Address coordinator_addr); storage::v3::Shard *db; @@ -188,26 +190,25 @@ struct InterpreterContext { const InterpreterConfig config; IdAllocator edge_ids_alloc; - // TODO (antaljanosbenjamin) Figure out an abstraction for io::Io to make it possible to construct an interpreter - // context with a simulator transport without templatizing it. - io::Io io; coordinator::Address coordinator_address; storage::v3::LabelId NameToLabelId(std::string_view label_name) { - return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name)); + return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name)); } storage::v3::PropertyId NameToPropertyId(std::string_view property_name) { - return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name)); + return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name)); } storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) { - return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name)); + return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name)); } + std::unique_ptr request_router_factory_; + private: // TODO Replace with local map of labels, properties and edge type ids - storage::v3::NameIdMapper query_id_mapper; + storage::v3::NameIdMapper query_id_mapper_; }; /// Function that is used to tell all active interpreters that they should stop @@ -296,7 +297,7 @@ class Interpreter final { */ void Abort(); - const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); } + RequestRouterInterface *GetRequestRouter() { return request_router_.get(); } private: struct QueryExecution { diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 996272fdc..de63cd76e 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -11,6 +11,7 @@ #pragma once +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include "coordinator/coordinator.hpp" @@ -31,6 +33,7 @@ #include "coordinator/shard_map.hpp" #include "io/address.hpp" #include "io/errors.hpp" +#include "io/local_transport/local_transport.hpp" #include "io/rsm/raft.hpp" #include "io/rsm/rsm_client.hpp" #include "io/rsm/shard_rsm.hpp" @@ -124,6 +127,10 @@ class RequestRouterInterface { virtual std::optional MaybeNameToLabel(const std::string &name) const = 0; virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0; virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0; + + virtual std::optional> AllocateInitialEdgeIds(io::Address coordinator_address) { + return {}; + } }; // TODO(kostasrim)rename this class template @@ -595,6 +602,23 @@ class RequestRouter : public RequestRouterInterface { edge_types_.StoreMapping(std::move(id_to_name)); } + std::optional> AllocateInitialEdgeIds(io::Address coordinator_address) override { + coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}}; + + io::rsm::WriteRequest ww; + ww.operation = requests; + auto resp = + io_.template Request, + io::rsm::WriteResponse>(coordinator_address, ww) + .Wait(); + if (resp.HasValue()) { + const auto alloc_edge_id_reps = + std::get(resp.GetValue().message.write_return); + return std::make_pair(alloc_edge_id_reps.low, alloc_edge_id_reps.high); + } + return {}; + } + ShardMap shards_map_; storage::v3::NameIdMapper properties_; storage::v3::NameIdMapper edge_types_; @@ -605,4 +629,89 @@ class RequestRouter : public RequestRouterInterface { coordinator::Hlc transaction_id_; // TODO(kostasrim) Add batch prefetching }; + +class RequestRouterFactory { + protected: + using LocalTransport = io::Io; + using SimulatorTransport = io::Io; + + using LocalTransportHandlePtr = std::shared_ptr; + using SimulatorTransportHandlePtr = std::shared_ptr; + + using TransportHandleVariant = std::variant; + + TransportHandleVariant transport_handle_; + + public: + explicit RequestRouterFactory(const TransportHandleVariant &transport_handle) : transport_handle_(transport_handle) {} + + RequestRouterFactory(const RequestRouterFactory &) = delete; + RequestRouterFactory &operator=(const RequestRouterFactory &) = delete; + RequestRouterFactory(RequestRouterFactory &&) = delete; + RequestRouterFactory &operator=(RequestRouterFactory &&) = delete; + + virtual ~RequestRouterFactory() = default; + + virtual TransportHandleVariant GetTransportHandle() { return transport_handle_; } + + virtual std::unique_ptr CreateRequestRouter( + const coordinator::Address &coordinator_address) const noexcept = 0; +}; + +class LocalRequestRouterFactory : public RequestRouterFactory { + public: + explicit LocalRequestRouterFactory(const TransportHandleVariant &transport_handle) + : RequestRouterFactory(transport_handle) {} + + std::unique_ptr CreateRequestRouter( + const coordinator::Address &coordinator_address) const noexcept override { + using TransportType = io::local_transport::LocalTransport; + auto actual_transport_handle = std::get(transport_handle_); + + boost::uuids::uuid random_uuid; + io::Address unique_local_addr_query; + + random_uuid = boost::uuids::uuid{boost::uuids::random_generator()()}; + unique_local_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); + + TransportType local_transport(actual_transport_handle); + auto local_transport_io = io::Io(local_transport, unique_local_addr_query); + + auto query_io = local_transport_io.ForkLocal(random_uuid); + + return std::make_unique>( + coordinator::CoordinatorClient(query_io, coordinator_address, {coordinator_address}), + std::move(local_transport_io)); + } +}; + +class SimulatedRequestRouterFactory : public RequestRouterFactory { + mutable io::simulator::Simulator *simulator_; + coordinator::Address address_; + + public: + explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator, coordinator::Address address) + : RequestRouterFactory(simulator.GetSimulatorHandle()), simulator_(&simulator), address_(address) {} + + std::unique_ptr CreateRequestRouter( + const coordinator::Address &coordinator_address) const noexcept override { + using TransportType = io::simulator::SimulatorTransport; + auto actual_transport_handle = std::get(transport_handle_); + + boost::uuids::uuid random_uuid; + io::Address unique_local_addr_query; + + // The simulated RR should not introduce stochastic behavior. + random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + auto io = simulator_->Register(unique_local_addr_query); + auto query_io = io.ForkLocal(random_uuid); + + return std::make_unique>( + coordinator::CoordinatorClient(query_io, coordinator_address, {coordinator_address}), + std::move(io)); + } +}; + } // namespace memgraph::query::v2 diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index 9e1a4c71e..f81f89798 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -17,7 +17,7 @@ function(add_simulation_test test_cpp) # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) - target_link_libraries(${target_name} mg-storage-v3 mg-communication mg-utils mg-io mg-io-simulator mg-coordinator mg-query-v2) + target_link_libraries(${target_name} mg-communication mg-utils mg-io mg-io-simulator mg-coordinator mg-query-v2 mg-storage-v3) target_link_libraries(${target_name} Boost::headers) target_link_libraries(${target_name} gtest gtest_main gmock rapidcheck rapidcheck_gtest) @@ -32,3 +32,4 @@ add_simulation_test(trial_query_storage/query_storage_test.cpp) add_simulation_test(sharded_map.cpp) add_simulation_test(shard_rsm.cpp) add_simulation_test(cluster_property_test.cpp) +add_simulation_test(cluster_property_test_v2.cpp) diff --git a/tests/simulation/cluster_property_test_v2.cpp b/tests/simulation/cluster_property_test_v2.cpp new file mode 100644 index 000000000..2f121647b --- /dev/null +++ b/tests/simulation/cluster_property_test_v2.cpp @@ -0,0 +1,65 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// This test serves as an example of a property-based model test. +// It generates a cluster configuration and a set of operations to +// apply against both the real system and a greatly simplified model. + +#include + +#include +#include +#include +#include + +#include "generated_operations.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/time.hpp" +#include "storage/v3/shard_manager.hpp" +#include "test_cluster.hpp" + +namespace memgraph::tests::simulation { + +using io::Duration; +using io::Time; +using io::simulator::SimulatorConfig; +using storage::v3::kMaximumCronInterval; + +RC_GTEST_PROP(RandomClusterConfig, HappyPath, (ClusterConfig cluster_config, NonEmptyOpVec ops, uint64_t rng_seed)) { + spdlog::cfg::load_env_levels(); + + SimulatorConfig sim_config{ + .drop_percent = 0, + .perform_timeouts = false, + .scramble_messages = true, + .rng_seed = rng_seed, + .start_time = Time::min(), + // TODO(tyler) set abort_time to something more restrictive than Time::max() + .abort_time = Time::max(), + }; + + std::vector queries = {"CREATE (n:test_label{property_1: 0, property_2: 0});", "MATCH (n) RETURN n;"}; + + auto [sim_stats_1, latency_stats_1] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries); + auto [sim_stats_2, latency_stats_2] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries); + + if (latency_stats_1 != latency_stats_2) { + spdlog::error("simulator stats diverged across runs"); + spdlog::error("run 1 simulator stats: {}", sim_stats_1); + spdlog::error("run 2 simulator stats: {}", sim_stats_2); + spdlog::error("run 1 latency:\n{}", latency_stats_1.SummaryTable()); + spdlog::error("run 2 latency:\n{}", latency_stats_2.SummaryTable()); + RC_ASSERT(latency_stats_1 == latency_stats_2); + RC_ASSERT(sim_stats_1 == sim_stats_2); + } +} + +} // namespace memgraph::tests::simulation diff --git a/tests/simulation/simulation_interpreter.hpp b/tests/simulation/simulation_interpreter.hpp new file mode 100644 index 000000000..b59f335be --- /dev/null +++ b/tests/simulation/simulation_interpreter.hpp @@ -0,0 +1,95 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include "io/simulator/simulator_handle.hpp" +#include "query/v2/config.hpp" +#include "query/v2/discard_value_stream.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/interpreter.hpp" +#include "query/v2/request_router.hpp" + +#include + +// TODO(gvolfing) +// -How to set up the entire raft cluster with the QE. Also provide abrstraction for that. +// -Pass an argument to the setup to determine, how many times the retry of a query should happen. + +namespace memgraph::io::simulator { + +class SimulatedInterpreter { + using ResultStream = query::v2::DiscardValueResultStream; + + public: + explicit SimulatedInterpreter(std::unique_ptr &&interpreter_context) + : interpreter_context_(std::move(interpreter_context)) { + interpreter_ = std::make_unique(&(*interpreter_context_)); + } + + SimulatedInterpreter(const SimulatedInterpreter &) = delete; + SimulatedInterpreter &operator=(const SimulatedInterpreter &) = delete; + SimulatedInterpreter(SimulatedInterpreter &&) = delete; + SimulatedInterpreter &operator=(SimulatedInterpreter &&) = delete; + ~SimulatedInterpreter() = default; + + std::vector RunQueries(const std::vector &queries) { + std::vector results; + results.reserve(queries.size()); + + for (const auto &query : queries) { + results.emplace_back(RunQuery(query)); + } + return results; + } + + private: + ResultStream RunQuery(const std::string &query) { + ResultStream stream; + + std::map params; + const std::string *username = nullptr; + + interpreter_->BeginTransaction(); + + auto *rr = interpreter_->GetRequestRouter(); + rr->StartTransaction(); + + interpreter_->Prepare(query, params, username); + interpreter_->PullAll(&stream); + interpreter_->CommitTransaction(); + + return stream; + } + + std::unique_ptr interpreter_context_; + std::unique_ptr interpreter_; +}; + +SimulatedInterpreter SetUpInterpreter(Address coordinator_address, Simulator &simulator) { + auto rr_factory = + std::make_unique(simulator, coordinator_address); + + auto interpreter_context = std::make_unique( + (memgraph::storage::v3::Shard *)(nullptr), + memgraph::query::v2::InterpreterConfig{.query = {.allow_load_csv = true}, + .execution_timeout_sec = 600, + .replication_replica_check_frequency = std::chrono::seconds(1), + .default_kafka_bootstrap_servers = "", + .default_pulsar_service_url = "", + .stream_transaction_conflict_retries = 30, + .stream_transaction_retry_interval = std::chrono::milliseconds(500)}, + std::filesystem::path("mg_data"), std::move(rr_factory), coordinator_address); + + return SimulatedInterpreter(std::move(interpreter_context)); +} + +} // namespace memgraph::io::simulator diff --git a/tests/simulation/test_cluster.hpp b/tests/simulation/test_cluster.hpp index 1392a0632..5aa792c16 100644 --- a/tests/simulation/test_cluster.hpp +++ b/tests/simulation/test_cluster.hpp @@ -36,6 +36,8 @@ #include "utils/print_helpers.hpp" #include "utils/variant_helpers.hpp" +#include "simulation_interpreter.hpp" + namespace memgraph::tests::simulation { using coordinator::Coordinator; @@ -277,4 +279,64 @@ std::pair RunClusterSimulation(const return std::make_pair(stats, histo); } +std::pair RunClusterSimulationWithQueries( + const SimulatorConfig &sim_config, const ClusterConfig &cluster_config, const std::vector &queries) { + spdlog::info("========================== NEW SIMULATION =========================="); + + auto simulator = Simulator(sim_config); + + auto machine_1_addr = Address::TestAddress(1); + auto cli_addr = Address::TestAddress(2); + auto cli_addr_2 = Address::TestAddress(3); + + Io cli_io = simulator.Register(cli_addr); + Io cli_io_2 = simulator.Register(cli_addr_2); + + auto coordinator_addresses = std::vector{ + machine_1_addr, + }; + + ShardMap initialization_sm = TestShardMap(cluster_config.shards - 1, cluster_config.replication_factor); + + auto mm_1 = MkMm(simulator, coordinator_addresses, machine_1_addr, initialization_sm); + Address coordinator_address = mm_1.CoordinatorAddress(); + + auto mm_thread_1 = std::jthread(RunMachine, std::move(mm_1)); + simulator.IncrementServerCountAndWaitForQuiescentState(machine_1_addr); + + auto detach_on_error = DetachIfDropped{.handle = mm_thread_1}; + + // TODO(tyler) clarify addresses of coordinator etc... as it's a mess + + CoordinatorClient coordinator_client(cli_io, coordinator_address, {coordinator_address}); + WaitForShardsToInitialize(coordinator_client); + + auto simulated_interpreter = io::simulator::SetUpInterpreter(coordinator_address, simulator); + + auto query_results = simulated_interpreter.RunQueries(queries); + + // We have now completed our workload without failing any assertions, so we can + // disable detaching the worker thread, which will cause the mm_thread_1 jthread + // to be joined when this function returns. + detach_on_error.detach = false; + + simulator.ShutDown(); + + mm_thread_1.join(); + + SimulatorStats stats = simulator.Stats(); + + spdlog::info("total messages: {}", stats.total_messages); + spdlog::info("dropped messages: {}", stats.dropped_messages); + spdlog::info("timed out requests: {}", stats.timed_out_requests); + spdlog::info("total requests: {}", stats.total_requests); + spdlog::info("total responses: {}", stats.total_responses); + spdlog::info("simulator ticks: {}", stats.simulator_ticks); + + auto histo = cli_io_2.ResponseLatencies(); + + spdlog::info("========================== SUCCESS :) =========================="); + return std::make_pair(stats, histo); +} + } // namespace memgraph::tests::simulation From f36b96744c317cce627a62c3521f642dd57176f2 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 15 Dec 2022 11:04:20 +0100 Subject: [PATCH 013/251] Apply post-merge fixes --- src/query/v2/plan/pretty_print.cpp | 4 ++-- src/query/v2/request_router.hpp | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/query/v2/plan/pretty_print.cpp b/src/query/v2/plan/pretty_print.cpp index adfab1a5f..762a1b19a 100644 --- a/src/query/v2/plan/pretty_print.cpp +++ b/src/query/v2/plan/pretty_print.cpp @@ -89,7 +89,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) { bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByPrimaryKey &op) { WithPrintLn([&](auto &out) { out << "* ScanAllByPrimaryKey" - << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")"; + << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")"; }); return true; } @@ -490,7 +490,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) { bool PlanToJsonVisitor::PreVisit(ScanAllByPrimaryKey &op) { json self; self["name"] = "ScanAllByPrimaryKey"; - self["label"] = ToJson(op.label_, *request_manager_); + self["label"] = ToJson(op.label_, *request_router_); self["output_symbol"] = ToJson(op.output_symbol_); op.input_->Accept(*this); diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 3dd2f164b..b5878ff92 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -114,6 +114,10 @@ class RequestRouterInterface { virtual std::optional MaybeNameToLabel(const std::string &name) const = 0; virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0; virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0; + // TODO - (gvolfing) Implement this function in the mocked class. + virtual std::vector GetSchemaForLabel(storage::v3::LabelId label) const { + return std::vector{}; + }; }; // TODO(kostasrim)rename this class template @@ -232,6 +236,10 @@ class RequestRouter : public RequestRouterInterface { }) != schema_it->second.end(); } + std::vector GetSchemaForLabel(storage::v3::LabelId label) const override { + return shards_map_.schemas.at(label); + } + bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); } // TODO(kostasrim) Simplify return result From af812d1311f75d936e1dd18c4dbec047ffc9c986 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 13 Dec 2022 09:05:39 +0100 Subject: [PATCH 014/251] Implement scanAll MultiFrame version --- src/query/v2/multiframe.hpp | 5 +- src/query/v2/plan/operator.cpp | 85 ++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index e13eb07ac..3e063bdde 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -201,10 +201,9 @@ class ValidFramesConsumer { ~ValidFramesConsumer() noexcept; ValidFramesConsumer(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete; + ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default; ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete; - + ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default; struct Iterator { using iterator_category = std::forward_iterator_tag; using difference_type = std::ptrdiff_t; diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 09c0837c0..804312800 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -37,6 +37,7 @@ #include "query/v2/db_accessor.hpp" #include "query/v2/exceptions.hpp" #include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/multiframe.hpp" #include "query/v2/path.hpp" #include "query/v2/plan/scoped_profile.hpp" #include "query/v2/request_router.hpp" @@ -473,11 +474,11 @@ class DistributedScanAllAndFilterCursor : public Cursor { if (label_.has_value()) { request_label = request_router.LabelToName(*label_); } - current_batch = request_router.ScanVertices(request_label); + current_batch_ = request_router.ScanVertices(request_label); } - current_vertex_it = current_batch.begin(); + current_vertex_it_ = current_batch_.begin(); request_state_ = State::COMPLETED; - return !current_batch.empty(); + return !current_batch_.empty(); } bool Pull(Frame &frame, ExecutionContext &context) override { @@ -495,23 +496,85 @@ class DistributedScanAllAndFilterCursor : public Cursor { } } - if (current_vertex_it == current_batch.end() && + if (current_vertex_it_ == current_batch_.end() && (request_state_ == State::COMPLETED || !MakeRequest(request_router, context))) { ResetExecutionState(); continue; } - frame[output_symbol_] = TypedValue(std::move(*current_vertex_it)); - ++current_vertex_it; + frame[output_symbol_] = TypedValue(std::move(*current_vertex_it_)); + ++current_vertex_it_; return true; } } + void Generate(ExecutionContext &context) { + auto &request_router = *context.request_router; + + input_cursor_->PullMultiple(*own_multi_frames_, context); + + if (!MakeRequest(request_router, context)) { + return; + } + + auto valid_frames_consumer = own_multi_frames_->GetValidFramesConsumer(); + auto valid_frames_it = valid_frames_consumer.begin(); + + for (auto valid_frames_it = valid_frames_consumer.begin(); valid_frames_it != valid_frames_consumer.end(); + ++valid_frames_it) { + for (auto vertex_it = current_batch_.begin(); vertex_it != current_batch_.end(); ++vertex_it) { + auto frame = *valid_frames_it; + frame[output_symbol_] = TypedValue(*vertex_it); + frames_buffer_.push(std::move(frame)); + } + valid_frames_it->MakeInvalid(); + } + } + + void PullMultiple(MultiFrame &input_multi_frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP(op_name_); + + if (!own_multi_frames_.has_value()) { + own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), + kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); + } + + auto &request_router = *context.request_router; + auto should_make_request = false; + auto should_pull = false; + auto should_generate = false; + + while (true) { + if (MustAbort(context)) { + throw HintedAbortError(); + } + + const auto should_generate = frames_buffer_.empty(); + if (should_generate) { + Generate(context); + } + + auto invalid_frames_populator = input_multi_frame.GetInvalidFramesPopulator(); + auto invalid_frame_it = invalid_frames_populator.begin(); + auto has_modified_at_least_one_frame = false; + while (invalid_frames_populator.end() != invalid_frame_it && !frames_buffer_.empty()) { + has_modified_at_least_one_frame = true; + *invalid_frame_it = frames_buffer_.front(); + ++invalid_frame_it; + frames_buffer_.pop(); + } + + if (!has_modified_at_least_one_frame) { + return; + } + } + }; + void Shutdown() override { input_cursor_->Shutdown(); } void ResetExecutionState() { - current_batch.clear(); - current_vertex_it = current_batch.end(); + current_batch_.clear(); + current_vertex_it_ = current_batch_.end(); request_state_ = State::INITIALIZING; } @@ -524,12 +587,14 @@ class DistributedScanAllAndFilterCursor : public Cursor { const Symbol output_symbol_; const UniqueCursorPtr input_cursor_; const char *op_name_; - std::vector current_batch; - std::vector::iterator current_vertex_it; + std::vector current_batch_; + std::vector::iterator current_vertex_it_; State request_state_ = State::INITIALIZING; std::optional label_; std::optional> property_expression_pair_; std::optional> filter_expressions_; + std::optional own_multi_frames_; + std::queue frames_buffer_; }; ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::v3::View view) From ac16348fff3d5f567d90faae944540b02b2821f4 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 13 Dec 2022 09:50:42 +0100 Subject: [PATCH 015/251] Remove unused variable --- src/query/v2/plan/operator.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 804312800..1569840c4 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -518,7 +518,6 @@ class DistributedScanAllAndFilterCursor : public Cursor { } auto valid_frames_consumer = own_multi_frames_->GetValidFramesConsumer(); - auto valid_frames_it = valid_frames_consumer.begin(); for (auto valid_frames_it = valid_frames_consumer.begin(); valid_frames_it != valid_frames_consumer.end(); ++valid_frames_it) { @@ -539,11 +538,6 @@ class DistributedScanAllAndFilterCursor : public Cursor { kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); } - auto &request_router = *context.request_router; - auto should_make_request = false; - auto should_pull = false; - auto should_generate = false; - while (true) { if (MustAbort(context)) { throw HintedAbortError(); From 83306d21defd278ca8f0bba7edc71b685338e832 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 13 Dec 2022 09:50:50 +0100 Subject: [PATCH 016/251] Revert changes --- src/query/v2/multiframe.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index 3e063bdde..d61755857 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -201,9 +201,9 @@ class ValidFramesConsumer { ~ValidFramesConsumer() noexcept; ValidFramesConsumer(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default; + ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete; ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default; + ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete; struct Iterator { using iterator_category = std::forward_iterator_tag; using difference_type = std::ptrdiff_t; From 54ce79baa02b281c8ca96d41c2c69dc0ef54c0d2 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 13 Dec 2022 09:51:42 +0100 Subject: [PATCH 017/251] Add empty line --- src/query/v2/multiframe.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index d61755857..e13eb07ac 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -204,6 +204,7 @@ class ValidFramesConsumer { ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete; ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete; ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete; + struct Iterator { using iterator_category = std::forward_iterator_tag; using difference_type = std::ptrdiff_t; From 311994a36ddda5ed7cddaff9fd364e3b96ba4f44 Mon Sep 17 00:00:00 2001 From: jeremy Date: Thu, 15 Dec 2022 14:31:43 +0100 Subject: [PATCH 018/251] Impl of version more memory friendly --- src/query/v2/multiframe.hpp | 6 ++-- src/query/v2/plan/operator.cpp | 54 +++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index e13eb07ac..aeacda7d3 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -200,9 +200,9 @@ class ValidFramesConsumer { explicit ValidFramesConsumer(MultiFrame &multiframe); ~ValidFramesConsumer() noexcept; - ValidFramesConsumer(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete; - ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete; + ValidFramesConsumer(const ValidFramesConsumer &other) = default; + ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default; + ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = default; ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete; struct Iterator { diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 1569840c4..3e8d533c5 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -508,26 +508,39 @@ class DistributedScanAllAndFilterCursor : public Cursor { } } - void Generate(ExecutionContext &context) { + void PrepareNextFrames(ExecutionContext &context) { auto &request_router = *context.request_router; input_cursor_->PullMultiple(*own_multi_frames_, context); + valid_frames_consumer_ = own_multi_frames_->GetValidFramesConsumer(); + valid_frames_it_ = valid_frames_consumer_->begin(); - if (!MakeRequest(request_router, context)) { - return; - } + MakeRequest(request_router, context); - auto valid_frames_consumer = own_multi_frames_->GetValidFramesConsumer(); + has_next_frame_ = current_vertex_it_ != current_batch_.end() && valid_frames_it_ != valid_frames_consumer_->end(); + } - for (auto valid_frames_it = valid_frames_consumer.begin(); valid_frames_it != valid_frames_consumer.end(); - ++valid_frames_it) { - for (auto vertex_it = current_batch_.begin(); vertex_it != current_batch_.end(); ++vertex_it) { - auto frame = *valid_frames_it; - frame[output_symbol_] = TypedValue(*vertex_it); - frames_buffer_.push(std::move(frame)); + inline bool HasNextFrame() { return has_next_frame_; } + + FrameWithValidity GetNextFrame(ExecutionContext &context) { + MG_ASSERT(HasNextFrame()); + + auto frame = *valid_frames_it_; + frame[output_symbol_] = TypedValue(*current_vertex_it_); + + ++current_vertex_it_; + if (current_vertex_it_ == current_batch_.end()) { + valid_frames_it_->MakeInvalid(); + ++valid_frames_it_; + + if (valid_frames_it_ == valid_frames_consumer_->end()) { + PrepareNextFrames(context); + } else { + current_vertex_it_ = current_batch_.begin(); } - valid_frames_it->MakeInvalid(); - } + }; + + return frame; } void PullMultiple(MultiFrame &input_multi_frame, ExecutionContext &context) override { @@ -536,6 +549,7 @@ class DistributedScanAllAndFilterCursor : public Cursor { if (!own_multi_frames_.has_value()) { own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); + PrepareNextFrames(context); } while (true) { @@ -543,19 +557,14 @@ class DistributedScanAllAndFilterCursor : public Cursor { throw HintedAbortError(); } - const auto should_generate = frames_buffer_.empty(); - if (should_generate) { - Generate(context); - } - auto invalid_frames_populator = input_multi_frame.GetInvalidFramesPopulator(); auto invalid_frame_it = invalid_frames_populator.begin(); auto has_modified_at_least_one_frame = false; - while (invalid_frames_populator.end() != invalid_frame_it && !frames_buffer_.empty()) { + + while (invalid_frames_populator.end() != invalid_frame_it && HasNextFrame()) { has_modified_at_least_one_frame = true; - *invalid_frame_it = frames_buffer_.front(); + *invalid_frame_it = GetNextFrame(context); ++invalid_frame_it; - frames_buffer_.pop(); } if (!has_modified_at_least_one_frame) { @@ -588,7 +597,10 @@ class DistributedScanAllAndFilterCursor : public Cursor { std::optional> property_expression_pair_; std::optional> filter_expressions_; std::optional own_multi_frames_; + std::optional valid_frames_consumer_; + ValidFramesConsumer::Iterator valid_frames_it_; std::queue frames_buffer_; + bool has_next_frame_; }; ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::v3::View view) From ae57fa31999a998676f750296ee7ec6c254eb334 Mon Sep 17 00:00:00 2001 From: gvolfing <107616712+gvolfing@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:25:46 +0100 Subject: [PATCH 019/251] Apply suggestions from code review Co-authored-by: Kostas Kyrimis --- src/io/local_transport/local_system.hpp | 2 +- src/io/simulator/simulator.hpp | 2 +- src/query/v2/interpreter.cpp | 6 +++--- src/query/v2/interpreter.hpp | 2 +- src/query/v2/request_router.hpp | 8 ++++---- tests/simulation/cluster_property_test_v2.cpp | 1 - tests/simulation/simulation_interpreter.hpp | 13 +++++++------ 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/io/local_transport/local_system.hpp b/src/io/local_transport/local_system.hpp index ec148b50b..feea44244 100644 --- a/src/io/local_transport/local_system.hpp +++ b/src/io/local_transport/local_system.hpp @@ -29,7 +29,7 @@ class LocalSystem { return Io{local_transport, address}; } - std::shared_ptr &GetTransportHandle() { return local_transport_handle_; } + std::shared_ptr GetTransportHandle() const { return local_transport_handle_; } void ShutDown() { local_transport_handle_->ShutDown(); } }; diff --git a/src/io/simulator/simulator.hpp b/src/io/simulator/simulator.hpp index 5095ab58b..87ced7180 100644 --- a/src/io/simulator/simulator.hpp +++ b/src/io/simulator/simulator.hpp @@ -50,6 +50,6 @@ class Simulator { SimulatorStats Stats() { return simulator_handle_->Stats(); } - std::shared_ptr GetSimulatorHandle() { return simulator_handle_; } + std::shared_ptr GetSimulatorHandle() const { return simulator_handle_; } }; }; // namespace memgraph::io::simulator diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index c428d1a39..8042a763c 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -793,7 +793,7 @@ using RWType = plan::ReadWriteTypeChecker::RWType; InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config, const std::filesystem::path & /*data_directory*/, - std::unique_ptr &&request_router_factory, + std::unique_ptr request_router_factory, coordinator::Address coordinator_addr) : db(db), config(config), @@ -807,10 +807,10 @@ Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_ interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address); // Get edge ids - const auto edge_ids_alloc_min_max_pair = + const auto [min, max] = request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address); if (edge_ids_alloc_min_max_pair) { - interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second}; + interpreter_context_->edge_ids_alloc = {min, max} } } diff --git a/src/query/v2/interpreter.hpp b/src/query/v2/interpreter.hpp index a83a26f11..05b05dd8e 100644 --- a/src/query/v2/interpreter.hpp +++ b/src/query/v2/interpreter.hpp @@ -173,7 +173,7 @@ struct PreparedQuery { struct InterpreterContext { explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config, const std::filesystem::path &data_directory, - std::unique_ptr &&request_router_factory, + std::unique_ptr request_router_factory, coordinator::Address coordinator_addr); storage::v3::Shard *db; diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index de63cd76e..7ac5d1816 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -655,7 +655,7 @@ class RequestRouterFactory { virtual TransportHandleVariant GetTransportHandle() { return transport_handle_; } virtual std::unique_ptr CreateRequestRouter( - const coordinator::Address &coordinator_address) const noexcept = 0; + const coordinator::Address &coordinator_address) const = 0; }; class LocalRequestRouterFactory : public RequestRouterFactory { @@ -664,7 +664,7 @@ class LocalRequestRouterFactory : public RequestRouterFactory { : RequestRouterFactory(transport_handle) {} std::unique_ptr CreateRequestRouter( - const coordinator::Address &coordinator_address) const noexcept override { + const coordinator::Address &coordinator_address) const override { using TransportType = io::local_transport::LocalTransport; auto actual_transport_handle = std::get(transport_handle_); @@ -686,7 +686,7 @@ class LocalRequestRouterFactory : public RequestRouterFactory { }; class SimulatedRequestRouterFactory : public RequestRouterFactory { - mutable io::simulator::Simulator *simulator_; + io::simulator::Simulator *simulator_; coordinator::Address address_; public: @@ -694,7 +694,7 @@ class SimulatedRequestRouterFactory : public RequestRouterFactory { : RequestRouterFactory(simulator.GetSimulatorHandle()), simulator_(&simulator), address_(address) {} std::unique_ptr CreateRequestRouter( - const coordinator::Address &coordinator_address) const noexcept override { + const coordinator::Address &coordinator_address) const override { using TransportType = io::simulator::SimulatorTransport; auto actual_transport_handle = std::get(transport_handle_); diff --git a/tests/simulation/cluster_property_test_v2.cpp b/tests/simulation/cluster_property_test_v2.cpp index 2f121647b..4996b4c2f 100644 --- a/tests/simulation/cluster_property_test_v2.cpp +++ b/tests/simulation/cluster_property_test_v2.cpp @@ -42,7 +42,6 @@ RC_GTEST_PROP(RandomClusterConfig, HappyPath, (ClusterConfig cluster_config, Non .scramble_messages = true, .rng_seed = rng_seed, .start_time = Time::min(), - // TODO(tyler) set abort_time to something more restrictive than Time::max() .abort_time = Time::max(), }; diff --git a/tests/simulation/simulation_interpreter.hpp b/tests/simulation/simulation_interpreter.hpp index b59f335be..e25dfaa99 100644 --- a/tests/simulation/simulation_interpreter.hpp +++ b/tests/simulation/simulation_interpreter.hpp @@ -9,8 +9,8 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -#include -#include +#include "machine_manager/machine_config.hpp" +#include "machine_manager/machine_manager.hpp" #include "io/simulator/simulator_handle.hpp" #include "query/v2/config.hpp" #include "query/v2/discard_value_stream.hpp" @@ -18,7 +18,8 @@ #include "query/v2/interpreter.hpp" #include "query/v2/request_router.hpp" -#include +#include +#include // TODO(gvolfing) // -How to set up the entire raft cluster with the QE. Also provide abrstraction for that. @@ -30,9 +31,9 @@ class SimulatedInterpreter { using ResultStream = query::v2::DiscardValueResultStream; public: - explicit SimulatedInterpreter(std::unique_ptr &&interpreter_context) + explicit SimulatedInterpreter(std::unique_ptr interpreter_context) : interpreter_context_(std::move(interpreter_context)) { - interpreter_ = std::make_unique(&(*interpreter_context_)); + interpreter_ = std::make_unique(interpreter_context_); } SimulatedInterpreter(const SimulatedInterpreter &) = delete; @@ -79,7 +80,7 @@ SimulatedInterpreter SetUpInterpreter(Address coordinator_address, Simulator &si std::make_unique(simulator, coordinator_address); auto interpreter_context = std::make_unique( - (memgraph::storage::v3::Shard *)(nullptr), + nullptr memgraph::query::v2::InterpreterConfig{.query = {.allow_load_csv = true}, .execution_timeout_sec = 600, .replication_replica_check_frequency = std::chrono::seconds(1), From 1aa40e5e3fad1e9b4a482157988ec8bd355a3ed0 Mon Sep 17 00:00:00 2001 From: jeremy Date: Thu, 15 Dec 2022 16:24:45 +0100 Subject: [PATCH 020/251] Add const to method --- src/query/v2/plan/operator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 3e8d533c5..b6bbf9ce1 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -520,7 +520,7 @@ class DistributedScanAllAndFilterCursor : public Cursor { has_next_frame_ = current_vertex_it_ != current_batch_.end() && valid_frames_it_ != valid_frames_consumer_->end(); } - inline bool HasNextFrame() { return has_next_frame_; } + inline bool HasNextFrame() const { return has_next_frame_; } FrameWithValidity GetNextFrame(ExecutionContext &context) { MG_ASSERT(HasNextFrame()); From fa39c6740b96d96bda49960f46656cc80519329d Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 15 Dec 2022 17:02:01 +0100 Subject: [PATCH 021/251] Apply review comments --- src/memgraph.cpp | 10 +++++----- src/query/v2/interpreter.cpp | 4 ++-- src/query/v2/request_router.hpp | 5 ++--- tests/simulation/CMakeLists.txt | 2 +- ...2.cpp => cluster_property_test_cypher_queries.cpp} | 0 tests/simulation/simulation_interpreter.hpp | 11 +++++------ 6 files changed, 15 insertions(+), 17 deletions(-) rename tests/simulation/{cluster_property_test_v2.cpp => cluster_property_test_cypher_queries.cpp} (100%) diff --git a/src/memgraph.cpp b/src/memgraph.cpp index ca03024fe..cdb1e63df 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -607,15 +607,15 @@ int main(int argc, char **argv) { // to minimize the impact of their failure on the main storage. memgraph::io::local_transport::LocalSystem ls; - auto unique_local_coord_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); - auto io = ls.Register(unique_local_coord_addr_query); + auto unique_local_addr_query = memgraph::coordinator::Address::UniqueLocalAddress(); + auto io = ls.Register(unique_local_addr_query); memgraph::machine_manager::MachineConfig config{ - .coordinator_addresses = std::vector{unique_local_coord_addr_query}, + .coordinator_addresses = std::vector{unique_local_addr_query}, .is_storage = true, .is_coordinator = true, - .listen_ip = unique_local_coord_addr_query.last_known_ip, - .listen_port = unique_local_coord_addr_query.last_known_port, + .listen_ip = unique_local_addr_query.last_known_ip, + .listen_port = unique_local_addr_query.last_known_port, }; memgraph::coordinator::ShardMap sm; diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index 8042a763c..46a479f36 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -807,10 +807,10 @@ Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_ interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address); // Get edge ids - const auto [min, max] = + const auto edge_ids_alloc_min_max_pair = request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address); if (edge_ids_alloc_min_max_pair) { - interpreter_context_->edge_ids_alloc = {min, max} + interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second}; } } diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 7ac5d1816..74117f52e 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -687,11 +687,10 @@ class LocalRequestRouterFactory : public RequestRouterFactory { class SimulatedRequestRouterFactory : public RequestRouterFactory { io::simulator::Simulator *simulator_; - coordinator::Address address_; public: - explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator, coordinator::Address address) - : RequestRouterFactory(simulator.GetSimulatorHandle()), simulator_(&simulator), address_(address) {} + explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator) + : RequestRouterFactory(simulator.GetSimulatorHandle()), simulator_(&simulator) {} std::unique_ptr CreateRequestRouter( const coordinator::Address &coordinator_address) const override { diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index f81f89798..28ff60b02 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -32,4 +32,4 @@ add_simulation_test(trial_query_storage/query_storage_test.cpp) add_simulation_test(sharded_map.cpp) add_simulation_test(shard_rsm.cpp) add_simulation_test(cluster_property_test.cpp) -add_simulation_test(cluster_property_test_v2.cpp) +add_simulation_test(cluster_property_test_cypher_queries.cpp) diff --git a/tests/simulation/cluster_property_test_v2.cpp b/tests/simulation/cluster_property_test_cypher_queries.cpp similarity index 100% rename from tests/simulation/cluster_property_test_v2.cpp rename to tests/simulation/cluster_property_test_cypher_queries.cpp diff --git a/tests/simulation/simulation_interpreter.hpp b/tests/simulation/simulation_interpreter.hpp index e25dfaa99..8e37f4f70 100644 --- a/tests/simulation/simulation_interpreter.hpp +++ b/tests/simulation/simulation_interpreter.hpp @@ -9,17 +9,17 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include "io/simulator/simulator_handle.hpp" #include "machine_manager/machine_config.hpp" #include "machine_manager/machine_manager.hpp" -#include "io/simulator/simulator_handle.hpp" #include "query/v2/config.hpp" #include "query/v2/discard_value_stream.hpp" #include "query/v2/frontend/ast/ast.hpp" #include "query/v2/interpreter.hpp" #include "query/v2/request_router.hpp" -#include #include +#include // TODO(gvolfing) // -How to set up the entire raft cluster with the QE. Also provide abrstraction for that. @@ -33,7 +33,7 @@ class SimulatedInterpreter { public: explicit SimulatedInterpreter(std::unique_ptr interpreter_context) : interpreter_context_(std::move(interpreter_context)) { - interpreter_ = std::make_unique(interpreter_context_); + interpreter_ = std::make_unique(interpreter_context_.get()); } SimulatedInterpreter(const SimulatedInterpreter &) = delete; @@ -76,11 +76,10 @@ class SimulatedInterpreter { }; SimulatedInterpreter SetUpInterpreter(Address coordinator_address, Simulator &simulator) { - auto rr_factory = - std::make_unique(simulator, coordinator_address); + auto rr_factory = std::make_unique(simulator); auto interpreter_context = std::make_unique( - nullptr + nullptr, memgraph::query::v2::InterpreterConfig{.query = {.allow_load_csv = true}, .execution_timeout_sec = 600, .replication_replica_check_frequency = std::chrono::seconds(1), From 32231fe49a7707bb61db4cf745f83fe0cc4b1862 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 15 Dec 2022 17:10:27 +0100 Subject: [PATCH 022/251] Move the implementation of AllocateInitialEdgeIds into the child class --- src/query/v2/request_router.hpp | 4 +--- tests/unit/query_v2_expression_evaluator.cpp | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 74117f52e..aeaf5740a 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -128,9 +128,7 @@ class RequestRouterInterface { virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0; virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0; - virtual std::optional> AllocateInitialEdgeIds(io::Address coordinator_address) { - return {}; - } + virtual std::optional> AllocateInitialEdgeIds(io::Address coordinator_address) = 0; }; // TODO(kostasrim)rename this class template diff --git a/tests/unit/query_v2_expression_evaluator.cpp b/tests/unit/query_v2_expression_evaluator.cpp index 5f77ed4e7..393952423 100644 --- a/tests/unit/query_v2_expression_evaluator.cpp +++ b/tests/unit/query_v2_expression_evaluator.cpp @@ -120,6 +120,10 @@ class MockedRequestRouter : public RequestRouterInterface { bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override { return true; } + std::optional> AllocateInitialEdgeIds(io::Address coordinator_address) override { + return {}; + } + private: void SetUpNameIdMappers() { std::unordered_map id_to_name; From bf21cbc9a9deec0618ce94089a63b49ae2e1c23f Mon Sep 17 00:00:00 2001 From: jbajic Date: Fri, 16 Dec 2022 09:13:07 +0100 Subject: [PATCH 023/251] Split vetrices, edges and transactions --- src/storage/v3/delta.hpp | 6 +++ src/storage/v3/shard.cpp | 82 +++++++++++++++++++++++++++++++--------- src/storage/v3/shard.hpp | 10 +++++ 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/storage/v3/delta.hpp b/src/storage/v3/delta.hpp index 1c6e57d2b..69da63e77 100644 --- a/src/storage/v3/delta.hpp +++ b/src/storage/v3/delta.hpp @@ -12,6 +12,9 @@ #pragma once #include + +#include + #include "storage/v3/edge_ref.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/property_value.hpp" @@ -129,6 +132,9 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { return !(a == b); } struct Delta { + // Needed for splits + boost::uuids::uuid uuid; + enum class Action { // Used for both Vertex and Edge DELETE_OBJECT, diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 294c99f65..305ef8650 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -23,6 +23,8 @@ #include "io/network/endpoint.hpp" #include "io/time.hpp" +#include "storage/v3/delta.hpp" +#include "storage/v3/edge.hpp" #include "storage/v3/edge_accessor.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/indices.hpp" @@ -1053,27 +1055,73 @@ std::optional Shard::ShouldSplit() const noexcept { return std::nullopt; } +void CollectDeltas(std::set &collected_transactions_start_id, Delta *delta) const { + while (delta != nullptr) { + collected_transactions_start_id.insert(delta->command_id); + delta = delta->next; + } +} + +VertexContainer Shard::CollectVertices(std::set &collected_transactions_start_id, + const PrimaryKey &split_key) { + VertexContainer splitted_data; + auto split_key_it = vertices_.find(split_key); + + for (; split_key_it != vertices_.end(); split_key_it++) { + // Go through deltas and pick up transactions start_id + CollectDeltas(collected_transactions_start_id, split_key_it->second.delta); + splitted_data.insert(vertices_.extract(split_key_it->first)); + } + return splitted_data; +} + +std::optional Shard::CollectEdges(std::set &collected_transactions_start_id, + const VertexContainer &split_vertices) const { + if (!config_.items.properties_on_edges) { + return std::nullopt; + } + EdgeContainer splitted_edges; + // TODO This copies edges without removing the unecessary ones!! + for (const auto &vertex : split_vertices) { + for (const auto &in_edge : vertex.second.in_edges) { + // This is safe since if properties_on_edges is true, the this must be a + // ptr + auto *edge = std::get<2>(in_edge).ptr; + CollectDeltas(collected_transactions_start_id, edge->delta); + + splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}}); + } + for (const auto &in_edge : vertex.second.out_edges) { + auto *edge = std::get<2>(in_edge).ptr; + CollectDeltas(collected_transactions_start_id, edge->delta); + + splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}}); + } + } + return splitted_edges; +} + +std::list Shard::CollectTransactions(const std::set &collected_transactions_start_id) const { + std::list transactions; + for (const auto commit_start : collected_transactions_start_id) { + transactions.push_back(*start_logical_id_to_transaction_[commit_start]); + } + return transactions; +} + SplitData Shard::PerformSplit(const PrimaryKey &split_key) { SplitData data; - data.vertices = std::map(vertices_.find(split_key), vertices_.end()); + std::set collected_transactions_start_id; + // Split Vertices + data.vertices = CollectVertices(collected_transactions_start_id, split_key); + // Resolve the deltas that were left on the shard, and are not referenced by + // neither of vertices + data.edges = CollectEdges(collected_transactions_start_id, data.vertices); data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; + // TODO Iterate over vertices and edges to replace their deltas with new ones tha are copied over + // use uuid - // Get all edges related with those vertices - if (config_.items.properties_on_edges) { - data.edges = std::invoke([&split_vertices = data.vertices]() { - // How to reserve? - EdgeContainer split_edges; - for (const auto &vertex : split_vertices) { - for (const auto &in_edge : vertex.second.in_edges) { - auto edge = std::get<2>(in_edge).ptr; - split_edges.insert(edge->gid, Edge{.gid = edge->gid, .delta = edge->delta, .properties = edge->properties}); - } - } - return split_edges; - }); - } - // TODO We also need to send ongoing transactions to the shard - // since they own deltas + data.transactions = CollectTransactions(collected_transactions_start_id); return data; } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 88849184d..ec9104ed6 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -185,6 +185,7 @@ struct SplitData { VertexContainer vertices; std::optional edges; IndicesInfo indices_info; + std::list transactions; }; /// Structure used to return information about the storage. @@ -374,6 +375,15 @@ class Shard final { SplitData PerformSplit(const PrimaryKey &split_key); private: + void CollectDeltas(std::set &collected_transactions_start_id, Delta *delta) const; + + std::list CollectTransactions(const std::set &collected_transactions_start_id) const; + + VertexContainer CollectVertices(std::set &collected_transactions_start_id, const PrimaryKey &split_key); + + std::optional CollectEdges(std::set &collected_transactions_start_id, + const VertexContainer &split_vertices) const; + Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level); uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); From 751c27f792d91d6a58d7601334eb17ae33295061 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 20 Dec 2022 10:12:50 +0100 Subject: [PATCH 024/251] Get ride of attribute has_valid_frames_ --- src/query/v2/plan/operator.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index b6bbf9ce1..f162ec9ea 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -516,11 +516,11 @@ class DistributedScanAllAndFilterCursor : public Cursor { valid_frames_it_ = valid_frames_consumer_->begin(); MakeRequest(request_router, context); - - has_next_frame_ = current_vertex_it_ != current_batch_.end() && valid_frames_it_ != valid_frames_consumer_->end(); } - inline bool HasNextFrame() const { return has_next_frame_; } + inline bool HasNextFrame() { + return current_vertex_it_ != current_batch_.end() && valid_frames_it_ != valid_frames_consumer_->end(); + } FrameWithValidity GetNextFrame(ExecutionContext &context) { MG_ASSERT(HasNextFrame()); @@ -600,7 +600,6 @@ class DistributedScanAllAndFilterCursor : public Cursor { std::optional valid_frames_consumer_; ValidFramesConsumer::Iterator valid_frames_it_; std::queue frames_buffer_; - bool has_next_frame_; }; ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::v3::View view) From 9589dd97b676896a062e3b2dc9f566de5524db8e Mon Sep 17 00:00:00 2001 From: jeremy Date: Fri, 30 Dec 2022 16:21:41 +0100 Subject: [PATCH 025/251] Impl and correct aggregate --- src/query/v2/multiframe.hpp | 2 +- src/query/v2/plan/operator.cpp | 116 +++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index aeacda7d3..0365b449f 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -168,7 +168,7 @@ class ValidFramesModifier { Iterator &operator++() { do { ptr_++; - } while (*this != iterator_wrapper_->end() && ptr_->IsValid()); + } while (*this != iterator_wrapper_->end() && !ptr_->IsValid()); return *this; } diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index b6bbf9ce1..43376aaed 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -1234,6 +1234,55 @@ class AggregateCursor : public Cursor { return true; } + void PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("AggregateMF"); + + if (!pulled_all_input_) { + while (!pulled_all_input_) { + ProcessAll(multi_frame, &context); + } + multi_frame.MakeAllFramesInvalid(); + aggregation_it_ = aggregation_.begin(); + + // in case there is no input and no group_bys we need to return true + // just this once + if (aggregation_.empty() && self_.group_by_.empty()) { + auto frame = multi_frame.GetFirstFrame(); + frame.MakeValid(); + auto *pull_memory = context.evaluation_context.memory; + // place default aggregation values on the frame + for (const auto &elem : self_.aggregations_) { + frame[elem.output_sym] = DefaultAggregationOpValue(elem, pull_memory); + } + // place null as remember values on the frame + for (const Symbol &remember_sym : self_.remember_) { + frame[remember_sym] = TypedValue(pull_memory); + } + return; + } + } + + if (aggregation_it_ == aggregation_.end()) { + return; + } + + // place aggregation values on the frame + auto &frame = multi_frame.GetFirstFrame(); + frame.MakeValid(); + auto aggregation_values_it = aggregation_it_->second.values_.begin(); + for (const auto &aggregation_elem : self_.aggregations_) { + frame[aggregation_elem.output_sym] = *aggregation_values_it++; + } + + // place remember values on the frame + auto remember_values_it = aggregation_it_->second.remember_.begin(); + for (const Symbol &remember_sym : self_.remember_) { + frame[remember_sym] = *remember_values_it++; + } + + aggregation_it_++; + } + void Shutdown() override { input_cursor_->Shutdown(); } void Reset() override { @@ -1312,6 +1361,36 @@ class AggregateCursor : public Cursor { } } + void ProcessAll(MultiFrame &multi_frame, ExecutionContext *context) { + input_cursor_->PullMultiple(multi_frame, *context); + auto valid_frames_modifier = + multi_frame.GetValidFramesConsumer(); // consumer is needed i.o. reader because of the evaluator + if (valid_frames_modifier.begin() == valid_frames_modifier.end()) { + // There are no valid frames, we stop + pulled_all_input_ = true; + return; + } + + for (auto &frame : valid_frames_modifier) { + ExpressionEvaluator evaluator(&frame, context->symbol_table, context->evaluation_context, context->request_router, + storage::v3::View::NEW); + ProcessOne(frame, &evaluator); + } + + // calculate AVG aggregations (so far they have only been summed) + for (size_t pos = 0; pos < self_.aggregations_.size(); ++pos) { + if (self_.aggregations_[pos].op != Aggregation::Op::AVG) continue; + for (auto &kv : aggregation_) { + AggregationValue &agg_value = kv.second; + auto count = agg_value.counts_[pos]; + auto *pull_memory = context->evaluation_context.memory; + if (count > 0) { + agg_value.values_[pos] = agg_value.values_[pos] / TypedValue(static_cast(count), pull_memory); + } + } + } + } + /** * Performs a single accumulation. */ @@ -1327,6 +1406,21 @@ class AggregateCursor : public Cursor { Update(evaluator, &agg_value); } + /** + * Performs a single accumulation. + */ + void ProcessOne(FrameWithValidity &frame, ExpressionEvaluator *evaluator) { + auto *mem = aggregation_.get_allocator().GetMemoryResource(); + utils::pmr::vector group_by(mem); + group_by.reserve(self_.group_by_.size()); + for (Expression *expression : self_.group_by_) { + group_by.emplace_back(expression->Accept(*evaluator)); + } + auto &agg_value = aggregation_.try_emplace(std::move(group_by), mem).first->second; + EnsureInitialized(frame, &agg_value); + Update(evaluator, &agg_value); + } + /** Ensures the new AggregationValue has been initialized. This means * that the value vectors are filled with an appropriate number of Nulls, * counts are set to 0 and remember values are remembered. @@ -1343,6 +1437,28 @@ class AggregateCursor : public Cursor { for (const Symbol &remember_sym : self_.remember_) agg_value->remember_.push_back(frame[remember_sym]); } + /** Ensures the new AggregationValue has been initialized. This means + * that the value vectors are filled with an appropriate number of Nulls, + * counts are set to 0 and remember values are remembered. + */ + void EnsureInitialized(FrameWithValidity &frame, AggregateCursor::AggregationValue *agg_value) const { + if (!agg_value->values_.empty()) { + frame.MakeInvalid(); + return; + } + + for (const auto &agg_elem : self_.aggregations_) { + auto *mem = agg_value->values_.get_allocator().GetMemoryResource(); + agg_value->values_.emplace_back(DefaultAggregationOpValue(agg_elem, mem)); + } + agg_value->counts_.resize(self_.aggregations_.size(), 0); + + for (const Symbol &remember_sym : self_.remember_) { + agg_value->remember_.push_back(frame[remember_sym]); + } + frame.MakeInvalid(); + } + /** Updates the given AggregationValue with new data. Assumes that * the AggregationValue has been initialized */ void Update(ExpressionEvaluator *evaluator, AggregateCursor::AggregationValue *agg_value) { From 68175bc97cf66fda671569f25880ed35f85e16c5 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 15 Dec 2022 15:20:32 +0100 Subject: [PATCH 026/251] Init basic cursor class Conform clang-tidy and modify PullMultiple behavior --- src/query/v2/multiframe.hpp | 8 +- src/query/v2/plan/operator.cpp | 167 ++++++++++++++++++++- src/query/v2/plan/pretty_print.hpp | 6 +- src/query/v2/plan/rewrite/index_lookup.hpp | 4 +- src/query/v2/request_router.hpp | 4 +- src/utils/event_counter.cpp | 1 + tests/unit/query_v2_plan.cpp | 24 ++- 7 files changed, 185 insertions(+), 29 deletions(-) diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index 8588993a7..49dad940a 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -200,9 +200,9 @@ class ValidFramesConsumer { explicit ValidFramesConsumer(MultiFrame &multiframe); ~ValidFramesConsumer() noexcept; - ValidFramesConsumer(const ValidFramesConsumer &other) = delete; - ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete; - ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete; + ValidFramesConsumer(const ValidFramesConsumer &other) = default; + ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default; + ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = default; ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete; struct Iterator { diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 7b8cfce07..02abdc3c4 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -462,6 +462,162 @@ class DistributedScanAllAndFilterCursor : public Cursor { std::optional> filter_expressions_; }; +class DistributedScanAllByPrimaryKeyCursor : public Cursor { + public: + explicit DistributedScanAllByPrimaryKeyCursor( + Symbol output_symbol, UniqueCursorPtr input_cursor, const char *op_name, + std::optional label, + std::optional> property_expression_pair, + std::optional> filter_expressions) + : output_symbol_(output_symbol), + input_cursor_(std::move(input_cursor)), + op_name_(op_name), + label_(label), + property_expression_pair_(property_expression_pair), + filter_expressions_(filter_expressions) { + ResetExecutionState(); + } + + enum class State : int8_t { INITIALIZING, COMPLETED }; + + using VertexAccessor = accessors::VertexAccessor; + + bool MakeRequest(RequestRouterInterface &request_router, ExecutionContext &context) { + { + SCOPED_REQUEST_WAIT_PROFILE; + std::optional request_label = std::nullopt; + if (label_.has_value()) { + request_label = request_router.LabelToName(*label_); + } + current_batch_ = request_router.ScanVertices(request_label); + } + current_vertex_it_ = current_batch_.begin(); + request_state_ = State::COMPLETED; + return !current_batch_.empty(); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP(op_name_); + + auto &request_router = *context.request_router; + while (true) { + if (MustAbort(context)) { + throw HintedAbortError(); + } + + if (request_state_ == State::INITIALIZING) { + if (!input_cursor_->Pull(frame, context)) { + return false; + } + } + + if (current_vertex_it_ == current_batch_.end() && + (request_state_ == State::COMPLETED || !MakeRequest(request_router, context))) { + ResetExecutionState(); + continue; + } + + frame[output_symbol_] = TypedValue(std::move(*current_vertex_it_)); + ++current_vertex_it_; + return true; + } + } + + void PrepareNextFrames(ExecutionContext &context) { + auto &request_router = *context.request_router; + + input_cursor_->PullMultiple(*own_multi_frames_, context); + valid_frames_consumer_ = own_multi_frames_->GetValidFramesConsumer(); + valid_frames_it_ = valid_frames_consumer_->begin(); + + MakeRequest(request_router, context); + } + + inline bool HasNextFrame() { + return current_vertex_it_ != current_batch_.end() && valid_frames_it_ != valid_frames_consumer_->end(); + } + + FrameWithValidity GetNextFrame(ExecutionContext &context) { + MG_ASSERT(HasNextFrame()); + + auto frame = *valid_frames_it_; + frame[output_symbol_] = TypedValue(*current_vertex_it_); + + ++current_vertex_it_; + if (current_vertex_it_ == current_batch_.end()) { + valid_frames_it_->MakeInvalid(); + ++valid_frames_it_; + + if (valid_frames_it_ == valid_frames_consumer_->end()) { + PrepareNextFrames(context); + } else { + current_vertex_it_ = current_batch_.begin(); + } + }; + + return frame; + } + + void PullMultiple(MultiFrame &input_multi_frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP(op_name_); + + if (!own_multi_frames_.has_value()) { + // NOLINTNEXTLINE(bugprone-narrowing-conversions) + own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), + kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); + PrepareNextFrames(context); + } + + while (true) { + if (MustAbort(context)) { + throw HintedAbortError(); + } + + auto invalid_frames_populator = input_multi_frame.GetInvalidFramesPopulator(); + auto invalid_frame_it = invalid_frames_populator.begin(); + auto has_modified_at_least_one_frame = false; + + while (invalid_frames_populator.end() != invalid_frame_it && HasNextFrame()) { + has_modified_at_least_one_frame = true; + *invalid_frame_it = GetNextFrame(context); + ++invalid_frame_it; + } + + if (!has_modified_at_least_one_frame) { + return; + } + } + }; + + void Shutdown() override { input_cursor_->Shutdown(); } + + void ResetExecutionState() { + current_batch_.clear(); + current_vertex_it_ = current_batch_.end(); + request_state_ = State::INITIALIZING; + } + + void Reset() override { + input_cursor_->Reset(); + ResetExecutionState(); + } + + private: + const Symbol output_symbol_; + const UniqueCursorPtr input_cursor_; + const char *op_name_; + std::vector current_batch_; + std::vector::iterator current_vertex_it_; + State request_state_ = State::INITIALIZING; + std::optional label_; + std::optional> property_expression_pair_; + std::optional> filter_expressions_; + std::optional own_multi_frames_; + std::optional valid_frames_consumer_; + ValidFramesConsumer::Iterator valid_frames_it_; + std::queue frames_buffer_; +}; + ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::v3::View view) : input_(input ? input : std::make_shared()), output_symbol_(output_symbol), view_(view) {} @@ -567,8 +723,13 @@ ScanAllByPrimaryKey::ScanAllByPrimaryKey(const std::shared_ptr ACCEPT_WITH_INPUT(ScanAllByPrimaryKey) -UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource * /*mem*/) const { - // EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); +UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); + + return MakeUniqueCursorPtr( + mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", std::nullopt /*label*/, + std::nullopt /*property_expression_pair*/, std::nullopt /*filter_expressions*/); + throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); } diff --git a/src/query/v2/plan/pretty_print.hpp b/src/query/v2/plan/pretty_print.hpp index 20fde180b..266b16102 100644 --- a/src/query/v2/plan/pretty_print.hpp +++ b/src/query/v2/plan/pretty_print.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -67,7 +67,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelProperty &) override; - bool PreVisit(query::v2::plan::ScanAllByPrimaryKey &) override; + bool PreVisit(ScanAllByPrimaryKey & /*unused*/) override; bool PreVisit(Expand &) override; bool PreVisit(ExpandVariable &) override; @@ -194,7 +194,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor { bool PreVisit(ScanAllByLabelPropertyRange &) override; bool PreVisit(ScanAllByLabelPropertyValue &) override; bool PreVisit(ScanAllByLabelProperty &) override; - bool PreVisit(ScanAllByPrimaryKey &) override; + bool PreVisit(ScanAllByPrimaryKey & /*unused*/) override; bool PreVisit(Produce &) override; bool PreVisit(Accumulate &) override; diff --git a/src/query/v2/plan/rewrite/index_lookup.hpp b/src/query/v2/plan/rewrite/index_lookup.hpp index 1a4a4160c..e134dd49b 100644 --- a/src/query/v2/plan/rewrite/index_lookup.hpp +++ b/src/query/v2/plan/rewrite/index_lookup.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -278,7 +278,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { return true; } - bool PostVisit(ScanAllByPrimaryKey &) override { + bool PostVisit(ScanAllByPrimaryKey & /*unused*/) override { prev_ops_.pop_back(); return true; } diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index b5878ff92..ca8d759f1 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -115,7 +115,7 @@ class RequestRouterInterface { virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0; virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0; // TODO - (gvolfing) Implement this function in the mocked class. - virtual std::vector GetSchemaForLabel(storage::v3::LabelId label) const { + virtual std::vector GetSchemaForLabel(storage::v3::LabelId /*label*/) const { return std::vector{}; }; }; diff --git a/src/utils/event_counter.cpp b/src/utils/event_counter.cpp index 634b6eae2..8f7475ffb 100644 --- a/src/utils/event_counter.cpp +++ b/src/utils/event_counter.cpp @@ -25,6 +25,7 @@ M(ScanAllByLabelPropertyValueOperator, "Number of times ScanAllByLabelPropertyValue operator was used.") \ M(ScanAllByLabelPropertyOperator, "Number of times ScanAllByLabelProperty operator was used.") \ M(ScanAllByIdOperator, "Number of times ScanAllById operator was used.") \ + M(ScanAllByPrimaryKeyOperator, "Number of times ScanAllByPrimaryKey operator was used.") \ M(ExpandOperator, "Number of times Expand operator was used.") \ M(ExpandVariableOperator, "Number of times ExpandVariable operator was used.") \ M(ConstructNamedPathOperator, "Number of times ConstructNamedPath operator was used.") \ diff --git a/tests/unit/query_v2_plan.cpp b/tests/unit/query_v2_plan.cpp index 2eb8295db..54d091df7 100644 --- a/tests/unit/query_v2_plan.cpp +++ b/tests/unit/query_v2_plan.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -86,11 +86,12 @@ class TestPlanner : public ::testing::Test {}; using PlannerTypes = ::testing::Types; -void DeleteListContent(std::list *list) { - for (BaseOpChecker *ptr : *list) { - delete ptr; - } -} +// void DeleteListContent(std::list *list) { +// for (BaseOpChecker *ptr : *list) { +// delete ptr; +// } +// } + TYPED_TEST_CASE(TestPlanner, PlannerTypes); TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { @@ -171,8 +172,8 @@ TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { dba.SetIndexCount(label, sec_prop_three.second, 1); memgraph::query::v2::AstStorage storage; - memgraph::query::v2::Expression *expected_primary_key; - expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); + // memgraph::query::v2::Expression *expected_primary_key; + // expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", prim_label_name))), WHERE(EQ(PROPERTY_LOOKUP("n", prim_prop_one), LITERAL(1))), RETURN("n"))); auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); @@ -183,12 +184,6 @@ TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { } /* - - - - - - TYPED_TEST(TestPlanner, MatchNodeReturn) { // Test MATCH (n) RETURN n AstStorage storage; @@ -1705,5 +1700,4 @@ TYPED_TEST(TestPlanner, Foreach) { } } */ - } // namespace From 7ef4114835e55d911dbc5ee79e5a26e14673ba56 Mon Sep 17 00:00:00 2001 From: jbajic Date: Mon, 9 Jan 2023 16:11:10 +0100 Subject: [PATCH 027/251] Fix splitting of edges --- src/storage/v3/shard.cpp | 78 +++++++++++++++++++-------------- src/storage/v3/shard.hpp | 34 +++++++++++--- src/storage/v3/shard_worker.hpp | 14 +++--- src/storage/v3/transaction.hpp | 18 +++++++- 4 files changed, 98 insertions(+), 46 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 305ef8650..86529ad5a 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -19,6 +19,7 @@ #include #include +#include #include #include "io/network/endpoint.hpp" @@ -1055,7 +1056,7 @@ std::optional Shard::ShouldSplit() const noexcept { return std::nullopt; } -void CollectDeltas(std::set &collected_transactions_start_id, Delta *delta) const { +void Shard::ScanDeltas(std::set &collected_transactions_start_id, Delta *delta) const { while (delta != nullptr) { collected_transactions_start_id.insert(delta->command_id); delta = delta->next; @@ -1069,59 +1070,72 @@ VertexContainer Shard::CollectVertices(std::set &collected_transaction for (; split_key_it != vertices_.end(); split_key_it++) { // Go through deltas and pick up transactions start_id - CollectDeltas(collected_transactions_start_id, split_key_it->second.delta); + ScanDeltas(collected_transactions_start_id, split_key_it->second.delta); splitted_data.insert(vertices_.extract(split_key_it->first)); } return splitted_data; } std::optional Shard::CollectEdges(std::set &collected_transactions_start_id, - const VertexContainer &split_vertices) const { + const VertexContainer &split_vertices, const PrimaryKey &split_key) { if (!config_.items.properties_on_edges) { return std::nullopt; } EdgeContainer splitted_edges; - // TODO This copies edges without removing the unecessary ones!! + const auto split_edges_from_vertex = [&](const auto &edges_ref) { + // This is safe since if properties_on_edges is true, the this must be a + // ptr + for (const auto &edge_ref : edges_ref) { + auto *edge = std::get<2>(edge_ref).ptr; + const auto &other_vtx = std::get<1>(edge_ref); + ScanDeltas(collected_transactions_start_id, edge->delta); + // Check if src and dest edge are both on splitted shard + // so we know if we should remove orphan edge + if (other_vtx.primary_key >= split_key) { + // Remove edge from shard + splitted_edges.insert(edges_.extract(edge->gid)); + } else { + splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}}); + } + } + }; + for (const auto &vertex : split_vertices) { - for (const auto &in_edge : vertex.second.in_edges) { - // This is safe since if properties_on_edges is true, the this must be a - // ptr - auto *edge = std::get<2>(in_edge).ptr; - CollectDeltas(collected_transactions_start_id, edge->delta); - - splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}}); - } - for (const auto &in_edge : vertex.second.out_edges) { - auto *edge = std::get<2>(in_edge).ptr; - CollectDeltas(collected_transactions_start_id, edge->delta); - - splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}}); - } + split_edges_from_vertex(vertex.second.in_edges); + split_edges_from_vertex(vertex.second.out_edges); } return splitted_edges; } -std::list Shard::CollectTransactions(const std::set &collected_transactions_start_id) const { - std::list transactions; - for (const auto commit_start : collected_transactions_start_id) { - transactions.push_back(*start_logical_id_to_transaction_[commit_start]); - } +std::map> Shard::CollectTransactions( + const std::set &collected_transactions_start_id) { + std::map> transactions; + // for (const auto commit_start : collected_transactions_start_id) { + // transactions.insert( + // {commit_start, std::make_unique(*start_logical_id_to_transaction_.at(commit_start))}); + // } return transactions; } SplitData Shard::PerformSplit(const PrimaryKey &split_key) { SplitData data; std::set collected_transactions_start_id; - // Split Vertices data.vertices = CollectVertices(collected_transactions_start_id, split_key); - // Resolve the deltas that were left on the shard, and are not referenced by - // neither of vertices - data.edges = CollectEdges(collected_transactions_start_id, data.vertices); - data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; - // TODO Iterate over vertices and edges to replace their deltas with new ones tha are copied over - // use uuid + data.edges = CollectEdges(collected_transactions_start_id, data.vertices, split_key); + // TODO indices wont work since timestamp cannot be replicated + // data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; - data.transactions = CollectTransactions(collected_transactions_start_id); + // data.transactions = CollectTransactions(collected_transactions_start_id); + + // Update delta addresses with the new addresses + // for (auto &vertex : data.vertices) { + // AdjustSplittedDataDeltas(vertex.second, data.transactions); + // } + // if (data.edges) { + // for (auto &edge : data.edges) { + // AdjustSplittedDataDeltas(edge, data.transactions); + // } + // } return data; } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index ec9104ed6..cab92eae9 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -185,7 +185,7 @@ struct SplitData { VertexContainer vertices; std::optional edges; IndicesInfo indices_info; - std::list transactions; + std::map transactions; }; /// Structure used to return information about the storage. @@ -375,14 +375,36 @@ class Shard final { SplitData PerformSplit(const PrimaryKey &split_key); private: - void CollectDeltas(std::set &collected_transactions_start_id, Delta *delta) const; + template + requires utils::SameAsAnyOf + void AdjustSplittedDataDeltas(TObj &delta_holder, const std::map &transactions) { + auto *delta_chain = delta_holder.delta; + Delta *new_delta_chain{nullptr}; + while (delta_chain != nullptr) { + auto &transaction = transactions.at(delta_chain->command_id); + // This is the address of corresponding delta + const auto transaction_delta_it = std::ranges::find_if( + transaction->deltas, [delta_uuid = delta_chain->uuid](const auto &elem) { return elem.uuid == delta_uuid; }); + // Add this delta to the new chain + if (new_delta_chain == nullptr) { + new_delta_chain = &*transaction_delta_it; + } else { + new_delta_chain->next = &*transaction_delta_it; + } + delta_chain = delta_chain->next; + } + delta_holder.delta = new_delta_chain; + } - std::list CollectTransactions(const std::set &collected_transactions_start_id) const; + void ScanDeltas(std::set &collected_transactions_start_id, Delta *delta) const; + + std::map> CollectTransactions( + const std::set &collected_transactions_start_id); VertexContainer CollectVertices(std::set &collected_transactions_start_id, const PrimaryKey &split_key); std::optional CollectEdges(std::set &collected_transactions_start_id, - const VertexContainer &split_vertices) const; + const VertexContainer &split_vertices, const PrimaryKey &split_key); Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level); @@ -391,7 +413,7 @@ class Shard final { // Main object storage NameIdMapper name_id_mapper_; LabelId primary_label_; - // The shard's range is [min, max) + // The shard's range is [min, max> PrimaryKey min_primary_key_; std::optional max_primary_key_; VertexContainer vertices_; diff --git a/src/storage/v3/shard_worker.hpp b/src/storage/v3/shard_worker.hpp index dcdc6ee13..e3d57964f 100644 --- a/src/storage/v3/shard_worker.hpp +++ b/src/storage/v3/shard_worker.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -174,12 +174,12 @@ class ShardWorker { Time next_for_uuid = rsm.Cron(); // Check if shard should split - if (const auto split_info = rsm.ShouldSplit(); split_info) { - // Request split from coordinator - // split_point => middle pk - // shard_id => uuid - // shard_version => - } + // if (const auto split_info = rsm.ShouldSplit(); split_info) { + // Request split from coordinator + // split_point => middle pk + // shard_id => uuid + // shard_version => + // } cron_schedule_.pop(); cron_schedule_.push(std::make_pair(next_for_uuid, uuid)); diff --git a/src/storage/v3/transaction.hpp b/src/storage/v3/transaction.hpp index 229e071b7..e0ed59290 100644 --- a/src/storage/v3/transaction.hpp +++ b/src/storage/v3/transaction.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -31,6 +31,16 @@ struct CommitInfo { }; struct Transaction { + Transaction(coordinator::Hlc start_timestamp, CommitInfo commit_info, uint64_t command_id, + const std::list &deltas, bool must_abort, bool is_aborted, IsolationLevel isolation_level) + : start_timestamp{start_timestamp}, + commit_info{std::make_unique(commit_info)}, + command_id(command_id), + deltas(CopyDeltas(deltas)), + must_abort(must_abort), + is_aborted(is_aborted), + isolation_level(isolation_level){}; + Transaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level) : start_timestamp(start_timestamp), commit_info(std::make_unique(CommitInfo{false, {start_timestamp}})), @@ -54,6 +64,12 @@ struct Transaction { ~Transaction() {} + std::list CopyDeltas(const std::list &deltas) const { return std::list{}; } + + Transaction Clone() const { + return {start_timestamp, *commit_info, command_id, deltas, must_abort, is_aborted, isolation_level}; + } + coordinator::Hlc start_timestamp; std::unique_ptr commit_info; uint64_t command_id; From 9dc16deae2ff9e6953631f0548a4816877fed85e Mon Sep 17 00:00:00 2001 From: jbajic Date: Tue, 10 Jan 2023 16:42:45 +0100 Subject: [PATCH 028/251] Copy deltas --- src/storage/v3/transaction.hpp | 52 ++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/storage/v3/transaction.hpp b/src/storage/v3/transaction.hpp index e0ed59290..6b208877a 100644 --- a/src/storage/v3/transaction.hpp +++ b/src/storage/v3/transaction.hpp @@ -31,12 +31,12 @@ struct CommitInfo { }; struct Transaction { - Transaction(coordinator::Hlc start_timestamp, CommitInfo commit_info, uint64_t command_id, - const std::list &deltas, bool must_abort, bool is_aborted, IsolationLevel isolation_level) + Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, uint64_t command_id, bool must_abort, + bool is_aborted, IsolationLevel isolation_level) : start_timestamp{start_timestamp}, - commit_info{std::make_unique(commit_info)}, + commit_info{std::make_unique(new_commit_info)}, command_id(command_id), - deltas(CopyDeltas(deltas)), + deltas(CopyDeltas(commit_info.get())), must_abort(must_abort), is_aborted(is_aborted), isolation_level(isolation_level){}; @@ -64,10 +64,50 @@ struct Transaction { ~Transaction() {} - std::list CopyDeltas(const std::list &deltas) const { return std::list{}; } + std::list CopyDeltas(CommitInfo *commit_info) const { + std::list copied_deltas; + for (const auto &delta : deltas) { + switch (delta.action) { + case Delta::Action::DELETE_OBJECT: + copied_deltas.emplace_back(Delta::DeleteObjectTag{}, commit_info, command_id); + break; + case Delta::Action::RECREATE_OBJECT: + copied_deltas.emplace_back(Delta::RecreateObjectTag{}, commit_info, command_id); + break; + case Delta::Action::ADD_LABEL: + copied_deltas.emplace_back(Delta::AddLabelTag{}, delta.label, commit_info, command_id); + break; + case Delta::Action::REMOVE_LABEL: + copied_deltas.emplace_back(Delta::RemoveLabelTag{}, delta.label, commit_info, command_id); + break; + case Delta::Action::ADD_IN_EDGE: + copied_deltas.emplace_back(Delta::AddInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, + delta.vertex_edge.edge, commit_info, command_id); + break; + case Delta::Action::ADD_OUT_EDGE: + copied_deltas.emplace_back(Delta::AddOutEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, + delta.vertex_edge.edge, commit_info, command_id); + break; + case Delta::Action::REMOVE_IN_EDGE: + copied_deltas.emplace_back(Delta::RemoveInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, + delta.vertex_edge.edge, commit_info, command_id); + break; + case Delta::Action::REMOVE_OUT_EDGE: + copied_deltas.emplace_back(Delta::RemoveOutEdgeTag{}, delta.vertex_edge.edge_type, + delta.vertex_edge.vertex_id, delta.vertex_edge.edge, commit_info, command_id); + break; + case Delta::Action::SET_PROPERTY: + copied_deltas.emplace_back(Delta::SetPropertyTag{}, delta.property.key, delta.property.value, commit_info, + command_id); + break; + } + } + return copied_deltas; + } + // This does not solve the whole problem of copying deltas Transaction Clone() const { - return {start_timestamp, *commit_info, command_id, deltas, must_abort, is_aborted, isolation_level}; + return {start_timestamp, *commit_info, command_id, must_abort, is_aborted, isolation_level}; } coordinator::Hlc start_timestamp; From 050d5efae73dbdb1537b7318276e6ff25825bb48 Mon Sep 17 00:00:00 2001 From: jbajic Date: Wed, 11 Jan 2023 14:16:41 +0100 Subject: [PATCH 029/251] Align deltas --- src/storage/v3/shard.cpp | 50 +++++++++++++++++++++++++++++----- src/storage/v3/shard.hpp | 8 ++++-- src/storage/v3/transaction.hpp | 1 + 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 86529ad5a..4ea89e6aa 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1107,13 +1107,49 @@ std::optional Shard::CollectEdges(std::set &collected_t return splitted_edges; } -std::map> Shard::CollectTransactions( - const std::set &collected_transactions_start_id) { - std::map> transactions; - // for (const auto commit_start : collected_transactions_start_id) { - // transactions.insert( - // {commit_start, std::make_unique(*start_logical_id_to_transaction_.at(commit_start))}); - // } +void Shard::AlignClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction, + std::map &cloned_transactions) { + // Align next and prev in deltas + // NOTE It is important that the order of delta lists is in same order + auto delta_it = transaction.deltas.begin(); + auto cloned_delta_it = cloned_transaction.deltas.begin(); + while (delta_it != transaction.deltas.end() && cloned_delta_it != cloned_transaction.deltas.end()) { + MG_ASSERT(delta_it->uuid == cloned_delta_it->uuid, "The order of deltas is not correct"); + // // We need to set prev and next on cloned_delta + // auto prev = delta_it->prev; + + // Find appropriate prev and delta->next for cloned deltas + auto *next = delta_it->next; + auto *cloned_next = &*cloned_delta_it; + while (next != nullptr) { + // No need to check we can be sure that it exists + cloned_next->next = &*std::ranges::find_if(cloned_transactions.at(next->command_id).deltas, + [next](const auto &delta) { return delta.uuid == next->uuid; }); + cloned_next = cloned_next->next; + next = next->next; + } + + ++delta_it; + ++cloned_delta_it; + } + MG_ASSERT(delta_it == transaction.deltas.end() && cloned_delta_it == cloned_transaction.deltas.end(), + "Both iterators must be exhausted!"); +} + +void Shard::AlignClonedTransactions(std::map &cloned_transactions) { + for (auto &[commit_start, cloned_transaction] : cloned_transactions) { + AlignClonedTransaction(cloned_transaction, *start_logical_id_to_transaction_[commit_start], cloned_transactions); + } +} + +std::map Shard::CollectTransactions(const std::set &collected_transactions_start_id) { + std::map transactions; + for (const auto commit_start : collected_transactions_start_id) { + transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()}); + } + // It is necessary to clone all the transactions first so we have new addresses + // for deltas, before doing alignment + AlignClonedTransactions(transactions); return transactions; } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index cab92eae9..502278564 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -398,8 +398,12 @@ class Shard final { void ScanDeltas(std::set &collected_transactions_start_id, Delta *delta) const; - std::map> CollectTransactions( - const std::set &collected_transactions_start_id); + void AlignClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction, + std::map &cloned_transactions); + + void AlignClonedTransactions(std::map &cloned_transactions); + + std::map CollectTransactions(const std::set &collected_transactions_start_id); VertexContainer CollectVertices(std::set &collected_transactions_start_id, const PrimaryKey &split_key); diff --git a/src/storage/v3/transaction.hpp b/src/storage/v3/transaction.hpp index 6b208877a..dba65067e 100644 --- a/src/storage/v3/transaction.hpp +++ b/src/storage/v3/transaction.hpp @@ -65,6 +65,7 @@ struct Transaction { ~Transaction() {} std::list CopyDeltas(CommitInfo *commit_info) const { + // TODO This does not solve the next and prev deltas that also need to be set std::list copied_deltas; for (const auto &delta : deltas) { switch (delta.action) { From 41bb988fe9e70a70dac418b16303866f8c4b6b81 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 14:14:59 +0100 Subject: [PATCH 030/251] Fix failing benchmark tests and implement cursor The benchmarking tests were failing because of the incorrect implementation of the ScanAllByPrimaryKeyCursor. The previous implementation caused the currently allocateable 1m edgeids to run out very quickly, causing the the tests to freeze. --- src/query/v2/context.hpp | 2 +- src/query/v2/interpreter.cpp | 2 +- src/query/v2/plan/operator.cpp | 40 ++++++++++++++++--- src/query/v2/request_router.hpp | 37 +++++++++++++++-- tests/mgbench/client.cpp | 2 +- tests/simulation/request_router.cpp | 4 +- tests/simulation/test_cluster.hpp | 4 +- tests/unit/high_density_shard_create_scan.cpp | 4 +- tests/unit/machine_manager.cpp | 4 +- tests/unit/query_v2_expression_evaluator.cpp | 7 +++- 10 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/query/v2/context.hpp b/src/query/v2/context.hpp index cb30a9ced..58f5ada97 100644 --- a/src/query/v2/context.hpp +++ b/src/query/v2/context.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index fde11ac00..9a4bfde90 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 02abdc3c4..b7841bf16 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -34,6 +34,7 @@ #include "query/v2/bindings/eval.hpp" #include "query/v2/bindings/symbol_table.hpp" #include "query/v2/context.hpp" +#include "query/v2/conversions.hpp" #include "query/v2/db_accessor.hpp" #include "query/v2/exceptions.hpp" #include "query/v2/frontend/ast/ast.hpp" @@ -403,7 +404,7 @@ class DistributedScanAllAndFilterCursor : public Cursor { if (label_.has_value()) { request_label = request_router.LabelToName(*label_); } - current_batch = request_router.ScanVertices(request_label); + current_batch = request_router.ScanVertices(request_label, std::nullopt); } current_vertex_it = current_batch.begin(); request_state_ = State::COMPLETED; @@ -468,13 +469,14 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { Symbol output_symbol, UniqueCursorPtr input_cursor, const char *op_name, std::optional label, std::optional> property_expression_pair, - std::optional> filter_expressions) + std::optional> filter_expressions, std::optional> primary_key) : output_symbol_(output_symbol), input_cursor_(std::move(input_cursor)), op_name_(op_name), label_(label), property_expression_pair_(property_expression_pair), - filter_expressions_(filter_expressions) { + filter_expressions_(filter_expressions), + primary_key_(primary_key) { ResetExecutionState(); } @@ -489,7 +491,32 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { if (label_.has_value()) { request_label = request_router.LabelToName(*label_); } - current_batch_ = request_router.ScanVertices(request_label); + current_batch_ = request_router.ScanVertices(request_label, std::nullopt); + } + current_vertex_it_ = current_batch_.begin(); + request_state_ = State::COMPLETED; + return !current_batch_.empty(); + } + + bool MakeRequestSingleFrame(Frame &frame, RequestRouterInterface &request_router, ExecutionContext &context) { + { + SCOPED_REQUEST_WAIT_PROFILE; + std::optional request_label = std::nullopt; + if (label_.has_value()) { + request_label = request_router.LabelToName(*label_); + } + + // Evaluate the expressions that hold the PrimaryKey. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.request_router, + storage::v3::View::OLD); + + std::vector pk; + MG_ASSERT(primary_key_); + for (auto primary_key : *primary_key_) { + pk.push_back(TypedValueToValue(primary_key->Accept(evaluator))); + } + + current_batch_ = request_router.ScanVertices(request_label, pk); } current_vertex_it_ = current_batch_.begin(); request_state_ = State::COMPLETED; @@ -612,6 +639,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { std::optional label_; std::optional> property_expression_pair_; std::optional> filter_expressions_; + std::optional> primary_key_; std::optional own_multi_frames_; std::optional valid_frames_consumer_; ValidFramesConsumer::Iterator valid_frames_it_; @@ -727,8 +755,8 @@ UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) cons EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); return MakeUniqueCursorPtr( - mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", std::nullopt /*label*/, - std::nullopt /*property_expression_pair*/, std::nullopt /*filter_expressions*/); + mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", label_, std::nullopt /*property_expression_pair*/, + std::nullopt /*filter_expressions*/, primary_key_); throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); } diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index ca8d759f1..934190cb4 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -97,7 +97,8 @@ class RequestRouterInterface { virtual void StartTransaction() = 0; virtual void Commit() = 0; - virtual std::vector ScanVertices(std::optional label) = 0; + virtual std::vector ScanVertices(std::optional label, + std::optional> primary_key) = 0; virtual std::vector CreateVertices(std::vector new_vertices) = 0; virtual std::vector ExpandOne(msgs::ExpandOneRequest request) = 0; virtual std::vector CreateExpand(std::vector new_edges) = 0; @@ -243,9 +244,16 @@ class RequestRouter : public RequestRouterInterface { bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); } // TODO(kostasrim) Simplify return result - std::vector ScanVertices(std::optional label) override { + std::vector ScanVertices(std::optional label, + std::optional> primary_key) override { // create requests - std::vector> requests_to_be_sent = RequestsForScanVertices(label); + std::vector> requests_to_be_sent; + if (primary_key) { + requests_to_be_sent = RequestsForScanVertexByPrimaryKey(label, *primary_key); + } else { + requests_to_be_sent = RequestsForScanVertices(label); + } + spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size()); // begin all requests in parallel @@ -510,6 +518,29 @@ class RequestRouter : public RequestRouterInterface { return requests; } + std::vector> RequestsForScanVertexByPrimaryKey( + const std::optional &label, const std::vector &primary_key) { + const auto label_id = shards_map_.GetLabelId(*label); + MG_ASSERT(label_id); + MG_ASSERT(IsPrimaryLabel(*label_id)); + std::vector> requests = {}; + + auto pk_containing_shard = + shards_map_.GetShardForKey(*label, storage::conversions::ConvertPropertyVector(primary_key)); + + msgs::ScanVerticesRequest request; + request.transaction_id = transaction_id_; + request.batch_limit = 1; + request.start_id.second = primary_key; + + ShardRequestState shard_request_state{ + .shard = pk_containing_shard, + .request = std::move(request), + }; + requests.emplace_back(std::move(shard_request_state)); + return requests; + } + std::vector> RequestsForExpandOne(const msgs::ExpandOneRequest &request) { std::map per_shard_request_table; msgs::ExpandOneRequest top_level_rqst_template = request; diff --git a/tests/mgbench/client.cpp b/tests/mgbench/client.cpp index e4b63d477..000c199fa 100644 --- a/tests/mgbench/client.cpp +++ b/tests/mgbench/client.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source diff --git a/tests/simulation/request_router.cpp b/tests/simulation/request_router.cpp index 4248e7876..10976baa9 100644 --- a/tests/simulation/request_router.cpp +++ b/tests/simulation/request_router.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -154,7 +154,7 @@ void RunStorageRaft(Raft &request_router, std void ExecuteOp(query::v2::RequestRouter &request_router, std::set &correctness_model, ScanAll scan_all) { - auto results = request_router.ScanVertices("test_label"); + auto results = request_router.ScanVertices("test_label", std::nullopt); RC_ASSERT(results.size() == correctness_model.size()); diff --git a/tests/unit/high_density_shard_create_scan.cpp b/tests/unit/high_density_shard_create_scan.cpp index 9fabf6ccc..a9647c795 100644 --- a/tests/unit/high_density_shard_create_scan.cpp +++ b/tests/unit/high_density_shard_create_scan.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -192,7 +192,7 @@ void ExecuteOp(query::v2::RequestRouter &request_router, std::se void ExecuteOp(query::v2::RequestRouter &request_router, std::set &correctness_model, ScanAll scan_all) { - auto results = request_router.ScanVertices("test_label"); + auto results = request_router.ScanVertices("test_label", std::nullopt); spdlog::error("got {} results, model size is {}", results.size(), correctness_model.size()); EXPECT_EQ(results.size(), correctness_model.size()); diff --git a/tests/unit/machine_manager.cpp b/tests/unit/machine_manager.cpp index 74b7d3863..8d0670e81 100644 --- a/tests/unit/machine_manager.cpp +++ b/tests/unit/machine_manager.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -111,7 +111,7 @@ ShardMap TestShardMap() { template void TestScanAll(RequestRouter &request_router) { - auto result = request_router.ScanVertices(kLabelName); + auto result = request_router.ScanVertices(kLabelName, std::nullopt); EXPECT_EQ(result.size(), 2); } diff --git a/tests/unit/query_v2_expression_evaluator.cpp b/tests/unit/query_v2_expression_evaluator.cpp index 5e91d0d5a..fb875a0f2 100644 --- a/tests/unit/query_v2_expression_evaluator.cpp +++ b/tests/unit/query_v2_expression_evaluator.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -84,7 +84,10 @@ class MockedRequestRouter : public RequestRouterInterface { } void StartTransaction() override {} void Commit() override {} - std::vector ScanVertices(std::optional /* label */) override { return {}; } + std::vector ScanVertices(std::optional /* label */, + std::optional> /*primary_key*/) override { + return {}; + } std::vector CreateVertices( std::vector /* new_vertices */) override { From 282725fd0f9258e34a07d22f29cc671870aec4ff Mon Sep 17 00:00:00 2001 From: jbajic Date: Thu, 12 Jan 2023 14:23:19 +0100 Subject: [PATCH 031/251] Adjust pointers --- src/storage/v3/shard.cpp | 77 +++++++++++++++++++++++++--------------- src/storage/v3/shard.hpp | 9 +++-- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 4ea89e6aa..856906385 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1108,25 +1108,50 @@ std::optional Shard::CollectEdges(std::set &collected_t } void Shard::AlignClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction, - std::map &cloned_transactions) { + std::map &cloned_transactions, + VertexContainer &cloned_vertices, EdgeContainer &cloned_edges) { // Align next and prev in deltas // NOTE It is important that the order of delta lists is in same order auto delta_it = transaction.deltas.begin(); auto cloned_delta_it = cloned_transaction.deltas.begin(); while (delta_it != transaction.deltas.end() && cloned_delta_it != cloned_transaction.deltas.end()) { MG_ASSERT(delta_it->uuid == cloned_delta_it->uuid, "The order of deltas is not correct"); - // // We need to set prev and next on cloned_delta - // auto prev = delta_it->prev; - // Find appropriate prev and delta->next for cloned deltas - auto *next = delta_it->next; - auto *cloned_next = &*cloned_delta_it; - while (next != nullptr) { - // No need to check we can be sure that it exists - cloned_next->next = &*std::ranges::find_if(cloned_transactions.at(next->command_id).deltas, - [next](const auto &delta) { return delta.uuid == next->uuid; }); - cloned_next = cloned_next->next; - next = next->next; + // auto *prev = &delta_it->prev; + // auto *cloned_prev = &cloned_delta_it->prev; + + auto *delta = &*delta_it; + auto *cloned_delta = &*cloned_delta_it; + while (delta != nullptr) { + // Align delta + cloned_delta->next = &*std::ranges::find_if(cloned_transactions.at(delta->command_id).deltas, + [delta](const auto &elem) { return elem.uuid == delta->uuid; }); + // Align prev ptr + auto ptr = delta->prev.Get(); + switch (ptr.type) { + case PreviousPtr::Type::NULLPTR: { + // noop + break; + } + case PreviousPtr::Type::DELTA: { + cloned_delta->prev.Set(ptr.delta); + break; + } + case PreviousPtr::Type::VERTEX: { + auto *cloned_vertex = &*cloned_vertices.find(ptr.vertex->first); + cloned_delta->prev.Set(cloned_vertex); + break; + } + case PreviousPtr::Type::EDGE: { + // TODO Case when there are no properties on edge is not handled + auto *cloned_edge = &*cloned_edges.find(ptr.edge->gid); + cloned_delta->prev.Set(&cloned_edge->second); + break; + } + }; + + cloned_delta = cloned_delta->next; + delta = delta->next; } ++delta_it; @@ -1136,20 +1161,24 @@ void Shard::AlignClonedTransaction(Transaction &cloned_transaction, const Transa "Both iterators must be exhausted!"); } -void Shard::AlignClonedTransactions(std::map &cloned_transactions) { +void Shard::AlignClonedTransactions(std::map &cloned_transactions, + VertexContainer &cloned_vertices, EdgeContainer &cloned_edges) { for (auto &[commit_start, cloned_transaction] : cloned_transactions) { - AlignClonedTransaction(cloned_transaction, *start_logical_id_to_transaction_[commit_start], cloned_transactions); + AlignClonedTransaction(cloned_transaction, *start_logical_id_to_transaction_[commit_start], cloned_transactions, + cloned_vertices, cloned_edges); } } -std::map Shard::CollectTransactions(const std::set &collected_transactions_start_id) { +std::map Shard::CollectTransactions(const std::set &collected_transactions_start_id, + VertexContainer &cloned_vertices, + EdgeContainer &cloned_edges) { std::map transactions; for (const auto commit_start : collected_transactions_start_id) { transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()}); } // It is necessary to clone all the transactions first so we have new addresses - // for deltas, before doing alignment - AlignClonedTransactions(transactions); + // for deltas, before doing alignment of deltas and prev_ptr + AlignClonedTransactions(transactions, cloned_vertices, cloned_edges); return transactions; } @@ -1158,21 +1187,11 @@ SplitData Shard::PerformSplit(const PrimaryKey &split_key) { std::set collected_transactions_start_id; data.vertices = CollectVertices(collected_transactions_start_id, split_key); data.edges = CollectEdges(collected_transactions_start_id, data.vertices, split_key); + data.transactions = CollectTransactions(collected_transactions_start_id, data.vertices, *data.edges); + // TODO indices wont work since timestamp cannot be replicated // data.indices_info = {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; - // data.transactions = CollectTransactions(collected_transactions_start_id); - - // Update delta addresses with the new addresses - // for (auto &vertex : data.vertices) { - // AdjustSplittedDataDeltas(vertex.second, data.transactions); - // } - // if (data.edges) { - // for (auto &edge : data.edges) { - // AdjustSplittedDataDeltas(edge, data.transactions); - // } - // } - return data; } diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 502278564..5a92253f4 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -399,11 +399,14 @@ class Shard final { void ScanDeltas(std::set &collected_transactions_start_id, Delta *delta) const; void AlignClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction, - std::map &cloned_transactions); + std::map &cloned_transactions, VertexContainer &cloned_vertices, + EdgeContainer &cloned_edges); - void AlignClonedTransactions(std::map &cloned_transactions); + void AlignClonedTransactions(std::map &cloned_transactions, VertexContainer &cloned_vertices, + EdgeContainer &cloned_edges); - std::map CollectTransactions(const std::set &collected_transactions_start_id); + std::map CollectTransactions(const std::set &collected_transactions_start_id, + VertexContainer &cloned_vertices, EdgeContainer &cloned_edges); VertexContainer CollectVertices(std::set &collected_transactions_start_id, const PrimaryKey &split_key); From afde0c69265ff64373c60940edaa6bc5f9fca7d4 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 15:45:14 +0100 Subject: [PATCH 032/251] Remove outcommented code, conform clang-tidy --- src/query/v2/plan/operator.cpp | 10 +- src/query/v2/plan/operator.lcp | 1 - tests/unit/query_common.hpp | 7 +- tests/unit/query_plan_checker.hpp | 6 +- tests/unit/query_plan_checker_v2.hpp | 166 +-- tests/unit/query_v2_plan.cpp | 1519 -------------------------- 6 files changed, 7 insertions(+), 1702 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index b7841bf16..b545b742f 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -512,7 +512,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { std::vector pk; MG_ASSERT(primary_key_); - for (auto primary_key : *primary_key_) { + for (auto *primary_key : *primary_key_) { pk.push_back(TypedValueToValue(primary_key->Accept(evaluator))); } @@ -589,7 +589,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { SCOPED_PROFILE_OP(op_name_); if (!own_multi_frames_.has_value()) { - // NOLINTNEXTLINE(bugprone-narrowing-conversions) + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); PrepareNextFrames(context); @@ -755,10 +755,8 @@ UniqueCursorPtr ScanAllByPrimaryKey::MakeCursor(utils::MemoryResource *mem) cons EventCounter::IncrementCounter(EventCounter::ScanAllByPrimaryKeyOperator); return MakeUniqueCursorPtr( - mem, output_symbol_, input_->MakeCursor(mem), "ScanAll", label_, std::nullopt /*property_expression_pair*/, - std::nullopt /*filter_expressions*/, primary_key_); - - throw QueryRuntimeException("ScanAllByPrimaryKey cursur is yet to be implemented."); + mem, output_symbol_, input_->MakeCursor(mem), "ScanAllByPrimaryKey", label_, + std::nullopt /*property_expression_pair*/, std::nullopt /*filter_expressions*/, primary_key_); } Expand::Expand(const std::shared_ptr &input, Symbol input_symbol, Symbol node_symbol, diff --git a/src/query/v2/plan/operator.lcp b/src/query/v2/plan/operator.lcp index f53425606..bc8b64e5d 100644 --- a/src/query/v2/plan/operator.lcp +++ b/src/query/v2/plan/operator.lcp @@ -868,7 +868,6 @@ given label and property. (:serialize (:slk)) (:clone)) - (lcp:define-struct expand-common () ( ;; info on what's getting expanded diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index f7fe6e805..d497b76ba 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -46,8 +46,6 @@ #include "storage/v2/id_types.hpp" #include "utils/string.hpp" -#include "query/v2/frontend/ast/ast.hpp" - namespace memgraph::query { namespace test_common { @@ -83,15 +81,12 @@ std::string ToString(NamedExpression *expr) { struct OrderBy { std::vector expressions; }; - struct Skip { Expression *expression = nullptr; }; - struct Limit { Expression *expression = nullptr; }; - struct OnMatch { std::vector set; }; diff --git a/tests/unit/query_plan_checker.hpp b/tests/unit/query_plan_checker.hpp index d04264c24..12ea2a683 100644 --- a/tests/unit/query_plan_checker.hpp +++ b/tests/unit/query_plan_checker.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -344,10 +344,6 @@ class ExpectScanAllByPrimaryKey : public OpChecker; - using ExpectCreateNode = OpChecker; using ExpectCreateExpand = OpChecker; using ExpectDelete = OpChecker; using ExpectScanAll = OpChecker; using ExpectScanAllByLabel = OpChecker; -// using ExpectScanAllById = OpChecker; using ExpectExpand = OpChecker; using ExpectFilter = OpChecker; using ExpectConstructNamedPath = OpChecker; @@ -157,120 +154,6 @@ using ExpectOrderBy = OpChecker; using ExpectUnwind = OpChecker; using ExpectDistinct = OpChecker; -// class ExpectForeach : public OpChecker { -// public: -// ExpectForeach(const std::list &input, const std::list &updates) -// : input_(input), updates_(updates) {} - -// void ExpectOp(Foreach &foreach, const SymbolTable &symbol_table) override { -// PlanChecker check_input(input_, symbol_table); -// foreach -// .input_->Accept(check_input); -// PlanChecker check_updates(updates_, symbol_table); -// foreach -// .update_clauses_->Accept(check_updates); -// } - -// private: -// std::list input_; -// std::list updates_; -// }; - -// class ExpectExpandVariable : public OpChecker { -// public: -// void ExpectOp(ExpandVariable &op, const SymbolTable &) override { -// EXPECT_EQ(op.type_, memgraph::query::EdgeAtom::Type::DEPTH_FIRST); -// } -// }; - -// class ExpectExpandBfs : public OpChecker { -// public: -// void ExpectOp(ExpandVariable &op, const SymbolTable &) override { -// EXPECT_EQ(op.type_, memgraph::query::EdgeAtom::Type::BREADTH_FIRST); -// } -// }; - -// class ExpectAccumulate : public OpChecker { -// public: -// explicit ExpectAccumulate(const std::unordered_set &symbols) : symbols_(symbols) {} - -// void ExpectOp(Accumulate &op, const SymbolTable &) override { -// std::unordered_set got_symbols(op.symbols_.begin(), op.symbols_.end()); -// EXPECT_EQ(symbols_, got_symbols); -// } - -// private: -// const std::unordered_set symbols_; -// }; - -// class ExpectAggregate : public OpChecker { -// public: -// ExpectAggregate(const std::vector &aggregations, -// const std::unordered_set &group_by) -// : aggregations_(aggregations), group_by_(group_by) {} - -// void ExpectOp(Aggregate &op, const SymbolTable &symbol_table) override { -// auto aggr_it = aggregations_.begin(); -// for (const auto &aggr_elem : op.aggregations_) { -// ASSERT_NE(aggr_it, aggregations_.end()); -// auto aggr = *aggr_it++; -// // TODO: Proper expression equality -// EXPECT_EQ(typeid(aggr_elem.value).hash_code(), typeid(aggr->expression1_).hash_code()); -// EXPECT_EQ(typeid(aggr_elem.key).hash_code(), typeid(aggr->expression2_).hash_code()); -// EXPECT_EQ(aggr_elem.op, aggr->op_); -// EXPECT_EQ(aggr_elem.output_sym, symbol_table.at(*aggr)); -// } -// EXPECT_EQ(aggr_it, aggregations_.end()); -// // TODO: Proper group by expression equality -// std::unordered_set got_group_by; -// for (auto *expr : op.group_by_) got_group_by.insert(typeid(*expr).hash_code()); -// std::unordered_set expected_group_by; -// for (auto *expr : group_by_) expected_group_by.insert(typeid(*expr).hash_code()); -// EXPECT_EQ(got_group_by, expected_group_by); -// } - -// private: -// std::vector aggregations_; -// std::unordered_set group_by_; -// }; - -// class ExpectMerge : public OpChecker { -// public: -// ExpectMerge(const std::list &on_match, const std::list &on_create) -// : on_match_(on_match), on_create_(on_create) {} - -// void ExpectOp(Merge &merge, const SymbolTable &symbol_table) override { -// PlanChecker check_match(on_match_, symbol_table); -// merge.merge_match_->Accept(check_match); -// PlanChecker check_create(on_create_, symbol_table); -// merge.merge_create_->Accept(check_create); -// } - -// private: -// const std::list &on_match_; -// const std::list &on_create_; -// }; - -// class ExpectOptional : public OpChecker { -// public: -// explicit ExpectOptional(const std::list &optional) : optional_(optional) {} - -// ExpectOptional(const std::vector &optional_symbols, const std::list &optional) -// : optional_symbols_(optional_symbols), optional_(optional) {} - -// void ExpectOp(Optional &optional, const SymbolTable &symbol_table) override { -// if (!optional_symbols_.empty()) { -// EXPECT_THAT(optional.optional_symbols_, testing::UnorderedElementsAreArray(optional_symbols_)); -// } -// PlanChecker check_optional(optional_, symbol_table); -// optional.optional_->Accept(check_optional); -// } - -// private: -// std::vector optional_symbols_; -// const std::list &optional_; -// }; - class ExpectScanAllByLabelPropertyValue : public OpChecker { public: ExpectScanAllByLabelPropertyValue(memgraph::storage::v3::LabelId label, @@ -291,53 +174,6 @@ class ExpectScanAllByLabelPropertyValue : public OpChecker { -// public: -// ExpectScanAllByLabelPropertyRange(memgraph::storage::LabelId label, memgraph::storage::PropertyId property, -// std::optional lower_bound, -// std::optional upper_bound) -// : label_(label), property_(property), lower_bound_(lower_bound), upper_bound_(upper_bound) {} - -// void ExpectOp(ScanAllByLabelPropertyRange &scan_all, const SymbolTable &) override { -// EXPECT_EQ(scan_all.label_, label_); -// EXPECT_EQ(scan_all.property_, property_); -// if (lower_bound_) { -// ASSERT_TRUE(scan_all.lower_bound_); -// // TODO: Proper expression equality -// EXPECT_EQ(typeid(scan_all.lower_bound_->value()).hash_code(), typeid(lower_bound_->value()).hash_code()); -// EXPECT_EQ(scan_all.lower_bound_->type(), lower_bound_->type()); -// } -// if (upper_bound_) { -// ASSERT_TRUE(scan_all.upper_bound_); -// // TODO: Proper expression equality -// EXPECT_EQ(typeid(scan_all.upper_bound_->value()).hash_code(), typeid(upper_bound_->value()).hash_code()); -// EXPECT_EQ(scan_all.upper_bound_->type(), upper_bound_->type()); -// } -// } - -// private: -// memgraph::storage::LabelId label_; -// memgraph::storage::PropertyId property_; -// std::optional lower_bound_; -// std::optional upper_bound_; -// }; - -// class ExpectScanAllByLabelProperty : public OpChecker { -// public: -// ExpectScanAllByLabelProperty(memgraph::storage::LabelId label, -// const std::pair &prop_pair) -// : label_(label), property_(prop_pair.second) {} - -// void ExpectOp(ScanAllByLabelProperty &scan_all, const SymbolTable &) override { -// EXPECT_EQ(scan_all.label_, label_); -// EXPECT_EQ(scan_all.property_, property_); -// } - -// private: -// memgraph::storage::LabelId label_; -// memgraph::storage::PropertyId property_; -// }; - class ExpectScanAllByPrimaryKey : public OpChecker { public: ExpectScanAllByPrimaryKey(memgraph::storage::v3::LabelId label, const std::vector &properties) diff --git a/tests/unit/query_v2_plan.cpp b/tests/unit/query_v2_plan.cpp index 54d091df7..56f90a690 100644 --- a/tests/unit/query_v2_plan.cpp +++ b/tests/unit/query_v2_plan.cpp @@ -172,8 +172,6 @@ TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { dba.SetIndexCount(label, sec_prop_three.second, 1); memgraph::query::v2::AstStorage storage; - // memgraph::query::v2::Expression *expected_primary_key; - // expected_primary_key = PROPERTY_LOOKUP("n", prim_prop_one); auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", prim_label_name))), WHERE(EQ(PROPERTY_LOOKUP("n", prim_prop_one), LITERAL(1))), RETURN("n"))); auto symbol_table = (memgraph::expr::MakeSymbolTable(query)); @@ -183,1521 +181,4 @@ TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { } } -/* -TYPED_TEST(TestPlanner, MatchNodeReturn) { - // Test MATCH (n) RETURN n - AstStorage storage; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(as_n))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CreateNodeReturn) { - // Test CREATE (n) RETURN n AS n - AstStorage storage; - auto ident_n = IDENT("n"); - auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), RETURN(ident_n, AS("n")))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CreateExpand) { - // Test CREATE (n) -[r :rel1]-> (m) - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "relationship"; - auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); - CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand()); -} - -TYPED_TEST(TestPlanner, CreateMultipleNode) { - // Test CREATE (n), (m) - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n")), PATTERN(NODE("m"))))); - CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateNode()); -} - -TYPED_TEST(TestPlanner, CreateNodeExpandNode) { - // Test CREATE (n) -[r :rel]-> (m), (l) - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "rel"; - auto *query = QUERY(SINGLE_QUERY( - CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m")), PATTERN(NODE("l"))))); - CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectCreateNode()); -} - -TYPED_TEST(TestPlanner, CreateNamedPattern) { - // Test CREATE p = (n) -[r :rel]-> (m) - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "rel"; - auto *query = - QUERY(SINGLE_QUERY(CREATE(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); - CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectConstructNamedPath()); -} - -TYPED_TEST(TestPlanner, MatchCreateExpand) { - // Test MATCH (n) CREATE (n) -[r :rel1]-> (m) - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "relationship"; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), - CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {relationship}), NODE("m"))))); - CheckPlan(query, storage, ExpectScanAll(), ExpectCreateExpand()); -} - -TYPED_TEST(TestPlanner, MatchLabeledNodes) { - // Test MATCH (n :label) RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = "label"; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", label))), RETURN(as_n))); - { - // Without created label index - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); - } - { - // With created label index - dba.SetIndexCount(dba.Label(label), 0); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectProduce()); - } -} - -TYPED_TEST(TestPlanner, MatchPathReturn) { - // Test MATCH (n) -[r :relationship]- (m) RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "relationship"; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), RETURN(as_n))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchNamedPatternReturn) { - // Test MATCH p = (n) -[r :relationship]- (m) RETURN p - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "relationship"; - auto *as_p = NEXPR("p", IDENT("p")); - auto *query = QUERY(SINGLE_QUERY( - MATCH(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), RETURN(as_p))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectConstructNamedPath(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchNamedPatternWithPredicateReturn) { - // Test MATCH p = (n) -[r :relationship]- (m) WHERE 2 = p RETURN p - AstStorage storage; - FakeDistributedDbAccessor dba; - auto relationship = "relationship"; - auto *as_p = NEXPR("p", IDENT("p")); - auto *query = - QUERY(SINGLE_QUERY(MATCH(NAMED_PATTERN("p", NODE("n"), EDGE("r", Direction::BOTH, {relationship}), NODE("m"))), - WHERE(EQ(LITERAL(2), IDENT("p"))), RETURN(as_p))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectConstructNamedPath(), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) { - // Test OPTIONAL MATCH p = (n) -[r]- (m) RETURN p - AstStorage storage; - auto node_n = NODE("n"); - auto edge = EDGE("r"); - auto node_m = NODE("m"); - auto pattern = NAMED_PATTERN("p", node_n, edge, node_m); - auto as_p = AS("p"); - auto *query = QUERY(SINGLE_QUERY(OPTIONAL_MATCH(pattern), RETURN("p", as_p))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto get_symbol = [&symbol_table](const auto *ast_node) { return symbol_table.at(*ast_node->identifier_); }; - std::vector optional_symbols{get_symbol(pattern), get_symbol(node_n), get_symbol(edge), get_symbol(node_m)}; - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - std::list optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()}; - CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce()); - DeleteListContent(&optional); -} - -TYPED_TEST(TestPlanner, MatchWhereReturn) { - // Test MATCH (n) WHERE n.property < 42 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto property = dba.Property("property"); - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WHERE(LESS(PROPERTY_LOOKUP("n", property), LITERAL(42))), RETURN(as_n))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchDelete) { - // Test MATCH (n) DELETE n - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), DELETE(IDENT("n")))); - CheckPlan(query, storage, ExpectScanAll(), ExpectDelete()); -} - -TYPED_TEST(TestPlanner, MatchNodeSet) { - // Test MATCH (n) SET n.prop = 42, n = n, n :label - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - auto label = "label"; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), SET(PROPERTY_LOOKUP("n", prop), LITERAL(42)), - SET("n", IDENT("n")), SET("n", {label}))); - CheckPlan(query, storage, ExpectScanAll(), ExpectSetProperty(), ExpectSetProperties(), ExpectSetLabels()); -} - -TYPED_TEST(TestPlanner, MatchRemove) { - // Test MATCH (n) REMOVE n.prop REMOVE n :label - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - auto label = "label"; - auto *query = - QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), REMOVE(PROPERTY_LOOKUP("n", prop)), REMOVE("n", {label}))); - CheckPlan(query, storage, ExpectScanAll(), ExpectRemoveProperty(), ExpectRemoveLabels()); -} - -TYPED_TEST(TestPlanner, MatchMultiPattern) { - // Test MATCH (n) -[r]- (m), (j) -[e]- (i) RETURN n - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY( - MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m")), PATTERN(NODE("j"), EDGE("e"), NODE("i"))), RETURN("n"))); - // We expect the expansions after the first to have a uniqueness filter in a - // single MATCH clause. - CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectScanAll(), ExpectExpand(), - ExpectEdgeUniquenessFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchMultiPatternSameStart) { - // Test MATCH (n), (n) -[e]- (m) RETURN n - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n")), PATTERN(NODE("n"), EDGE("e"), NODE("m"))), RETURN("n"))); - // We expect the second pattern to generate only an Expand, since another - // ScanAll would be redundant. - CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchMultiPatternSameExpandStart) { - // Test MATCH (n) -[r]- (m), (m) -[e]- (l) RETURN n - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY( - MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m")), PATTERN(NODE("m"), EDGE("e"), NODE("l"))), RETURN("n"))); - // We expect the second pattern to generate only an Expand. Another - // ScanAll would be redundant, as it would generate the nodes obtained from - // expansion. Additionally, a uniqueness filter is expected. - CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectExpand(), ExpectEdgeUniquenessFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MultiMatch) { - // Test MATCH (n) -[r]- (m) MATCH (j) -[e]- (i) -[f]- (h) RETURN n - AstStorage storage; - auto *node_n = NODE("n"); - auto *edge_r = EDGE("r"); - auto *node_m = NODE("m"); - auto *node_j = NODE("j"); - auto *edge_e = EDGE("e"); - auto *node_i = NODE("i"); - auto *edge_f = EDGE("f"); - auto *node_h = NODE("h"); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n, edge_r, node_m)), - MATCH(PATTERN(node_j, edge_e, node_i, edge_f, node_h)), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // Multiple MATCH clauses form a Cartesian product, so the uniqueness should - // not cross MATCH boundaries. - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectScanAll(), ExpectExpand(), - ExpectExpand(), ExpectEdgeUniquenessFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MultiMatchSameStart) { - // Test MATCH (n) MATCH (n) -[r]- (m) RETURN n - AstStorage storage; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = - QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), RETURN(as_n))); - // Similar to MatchMultiPatternSameStart, we expect only Expand from second - // MATCH clause. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchWithReturn) { - // Test MATCH (old) WITH old AS new RETURN new - AstStorage storage; - auto *as_new = NEXPR("new", IDENT("new")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("old"))), WITH("old", AS("new")), RETURN(as_new))); - // No accumulation since we only do reads. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchWithWhereReturn) { - // Test MATCH (old) WITH old AS new WHERE new.prop < 42 RETURN new - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *as_new = NEXPR("new", IDENT("new")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("old"))), WITH("old", AS("new")), - WHERE(LESS(PROPERTY_LOOKUP("new", prop), LITERAL(42))), RETURN(as_new))); - // No accumulation since we only do reads. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CreateMultiExpand) { - // Test CREATE (n) -[r :r]-> (m), (n) - [p :p]-> (l) - FakeDistributedDbAccessor dba; - auto r = "r"; - auto p = "p"; - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {r}), NODE("m")), - PATTERN(NODE("n"), EDGE("p", Direction::OUT, {p}), NODE("l"))))); - CheckPlan(query, storage, ExpectCreateNode(), ExpectCreateExpand(), ExpectCreateExpand()); -} - -TYPED_TEST(TestPlanner, MatchWithSumWhereReturn) { - // Test MATCH (n) WITH SUM(n.prop) + 42 AS sum WHERE sum < 42 - // RETURN sum AS result - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto sum = SUM(PROPERTY_LOOKUP("n", prop)); - auto literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WITH(ADD(sum, literal), AS("sum")), - WHERE(LESS(IDENT("sum"), LITERAL(42))), RETURN("sum", AS("result")))); - auto aggr = ExpectAggregate({sum}, {literal}); - CheckPlan(query, storage, ExpectScanAll(), aggr, ExpectProduce(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchReturnSum) { - // Test MATCH (n) RETURN SUM(n.prop1) AS sum, n.prop2 AS group - FakeDistributedDbAccessor dba; - auto prop1 = dba.Property("prop1"); - auto prop2 = dba.Property("prop2"); - AstStorage storage; - auto sum = SUM(PROPERTY_LOOKUP("n", prop1)); - auto n_prop2 = PROPERTY_LOOKUP("n", prop2); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(sum, AS("sum"), n_prop2, AS("group")))); - auto aggr = ExpectAggregate({sum}, {n_prop2}); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CreateWithSum) { - // Test CREATE (n) WITH SUM(n.prop) AS sum - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto ident_n = IDENT("n"); - auto n_prop = PROPERTY_LOOKUP(ident_n, prop); - auto sum = SUM(n_prop); - auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), WITH(sum, AS("sum")))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); - auto aggr = ExpectAggregate({sum}, {}); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect both the accumulation and aggregation because the part before - // WITH updates the database. - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchWithCreate) { - // Test MATCH (n) WITH n AS a CREATE (a) -[r :r]-> (b) - FakeDistributedDbAccessor dba; - auto r_type = "r"; - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WITH("n", AS("a")), - CREATE(PATTERN(NODE("a"), EDGE("r", Direction::OUT, {r_type}), NODE("b"))))); - CheckPlan(query, storage, ExpectScanAll(), ExpectProduce(), ExpectCreateExpand()); -} - -TYPED_TEST(TestPlanner, MatchReturnSkipLimit) { - // Test MATCH (n) RETURN n SKIP 2 LIMIT 1 - AstStorage storage; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), RETURN(as_n, SKIP(LITERAL(2)), LIMIT(LITERAL(1))))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectSkip(), ExpectLimit()); -} - -TYPED_TEST(TestPlanner, CreateWithSkipReturnLimit) { - // Test CREATE (n) WITH n AS m SKIP 2 RETURN m LIMIT 1 - AstStorage storage; - auto ident_n = IDENT("n"); - auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), WITH(ident_n, AS("m"), SKIP(LITERAL(2))), - RETURN("m", LIMIT(LITERAL(1))))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // Since we have a write query, we need to have Accumulate. This is a bit - // different than Neo4j 3.0, which optimizes WITH followed by RETURN as a - // single RETURN clause and then moves Skip and Limit before Accumulate. - // This causes different behaviour. A newer version of Neo4j does the same - // thing as us here (but who knows if they change it again). - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, ExpectProduce(), ExpectSkip(), ExpectProduce(), - ExpectLimit()); -} - -TYPED_TEST(TestPlanner, CreateReturnSumSkipLimit) { - // Test CREATE (n) RETURN SUM(n.prop) AS s SKIP 2 LIMIT 1 - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto ident_n = IDENT("n"); - auto n_prop = PROPERTY_LOOKUP(ident_n, prop); - auto sum = SUM(n_prop); - auto query = - QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"))), RETURN(sum, AS("s"), SKIP(LITERAL(2)), LIMIT(LITERAL(1))))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); - auto aggr = ExpectAggregate({sum}, {}); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce(), ExpectSkip(), ExpectLimit()); -} - -TYPED_TEST(TestPlanner, MatchReturnOrderBy) { - // Test MATCH (n) RETURN n AS m ORDER BY n.prop - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *as_m = NEXPR("m", IDENT("n")); - auto *node_n = NODE("n"); - auto ret = RETURN(as_m, ORDER_BY(PROPERTY_LOOKUP("n", prop))); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n)), ret)); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectProduce(), ExpectOrderBy()); -} - -TYPED_TEST(TestPlanner, CreateWithOrderByWhere) { - // Test CREATE (n) -[r :r]-> (m) - // WITH n AS new ORDER BY new.prop, r.prop WHERE m.prop < 42 - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - auto r_type = "r"; - AstStorage storage; - auto ident_n = IDENT("n"); - auto ident_r = IDENT("r"); - auto ident_m = IDENT("m"); - auto new_prop = PROPERTY_LOOKUP("new", prop); - auto r_prop = PROPERTY_LOOKUP(ident_r, prop); - auto m_prop = PROPERTY_LOOKUP(ident_m, prop); - auto query = - QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", Direction::OUT, {r_type}), NODE("m"))), - WITH(ident_n, AS("new"), ORDER_BY(new_prop, r_prop)), WHERE(LESS(m_prop, LITERAL(42))))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - // Since this is a write query, we expect to accumulate to old used symbols. - auto acc = ExpectAccumulate({ - symbol_table.at(*ident_n), // `n` in WITH - symbol_table.at(*ident_r), // `r` in ORDER BY - symbol_table.at(*ident_m), // `m` in WHERE - }); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), ExpectCreateExpand(), acc, ExpectProduce(), - ExpectOrderBy(), ExpectFilter()); -} - -TYPED_TEST(TestPlanner, ReturnAddSumCountOrderBy) { - // Test RETURN SUM(1) + COUNT(2) AS result ORDER BY result - AstStorage storage; - auto sum = SUM(LITERAL(1)); - auto count = COUNT(LITERAL(2)); - auto *query = QUERY(SINGLE_QUERY(RETURN(ADD(sum, count), AS("result"), ORDER_BY(IDENT("result"))))); - auto aggr = ExpectAggregate({sum, count}, {}); - CheckPlan(query, storage, aggr, ExpectProduce(), ExpectOrderBy()); -} - -TYPED_TEST(TestPlanner, MatchMerge) { - // Test MATCH (n) MERGE (n) -[r :r]- (m) - // ON MATCH SET n.prop = 42 ON CREATE SET m = n - // RETURN n AS n - FakeDistributedDbAccessor dba; - auto r_type = "r"; - auto prop = dba.Property("prop"); - AstStorage storage; - auto ident_n = IDENT("n"); - auto query = - QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), - MERGE(PATTERN(NODE("n"), EDGE("r", Direction::BOTH, {r_type}), NODE("m")), - ON_MATCH(SET(PROPERTY_LOOKUP("n", prop), LITERAL(42))), ON_CREATE(SET("m", IDENT("n")))), - RETURN(ident_n, AS("n")))); - std::list on_match{new ExpectExpand(), new ExpectSetProperty()}; - std::list on_create{new ExpectCreateExpand(), new ExpectSetProperties()}; - auto symbol_table = memgraph::query::MakeSymbolTable(query); - // We expect Accumulate after Merge, because it is considered as a write. - auto acc = ExpectAccumulate({symbol_table.at(*ident_n)}); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce()); - DeleteListContent(&on_match); - DeleteListContent(&on_create); -} - -TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) { - // Test MATCH (n) OPTIONAL MATCH (n) -[r]- (m) WHERE m.prop < 42 RETURN r - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), - WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r"))); - std::list optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()}; - CheckPlan(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce()); - DeleteListContent(&optional); -} - -TYPED_TEST(TestPlanner, MatchUnwindReturn) { - // Test MATCH (n) UNWIND [1,2,3] AS x RETURN n, x - AstStorage storage; - auto *as_n = NEXPR("n", IDENT("n")); - auto *as_x = NEXPR("x", IDENT("x")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), UNWIND(LIST(LITERAL(1), LITERAL(2), LITERAL(3)), AS("x")), - RETURN(as_n, as_x))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectUnwind(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ReturnDistinctOrderBySkipLimit) { - // Test RETURN DISTINCT 1 ORDER BY 1 SKIP 1 LIMIT 1 - AstStorage storage; - auto *query = QUERY( - SINGLE_QUERY(RETURN_DISTINCT(LITERAL(1), AS("1"), ORDER_BY(LITERAL(1)), SKIP(LITERAL(1)), LIMIT(LITERAL(1))))); - CheckPlan(query, storage, ExpectProduce(), ExpectDistinct(), ExpectOrderBy(), ExpectSkip(), ExpectLimit()); -} - -TYPED_TEST(TestPlanner, CreateWithDistinctSumWhereReturn) { - // Test CREATE (n) WITH DISTINCT SUM(n.prop) AS s WHERE s < 42 RETURN s - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto node_n = NODE("n"); - auto sum = SUM(PROPERTY_LOOKUP("n", prop)); - auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(node_n)), WITH_DISTINCT(sum, AS("s")), - WHERE(LESS(IDENT("s"), LITERAL(42))), RETURN("s"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto acc = ExpectAccumulate({symbol_table.at(*node_n->identifier_)}); - auto aggr = ExpectAggregate({sum}, {}); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectCreateNode(), acc, aggr, ExpectProduce(), ExpectDistinct(), - ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchCrossReferenceVariable) { - // Test MATCH (n {prop: m.prop}), (m {prop: n.prop}) RETURN n - FakeDistributedDbAccessor dba; - auto prop = PROPERTY_PAIR("prop"); - AstStorage storage; - auto node_n = NODE("n"); - auto m_prop = PROPERTY_LOOKUP("m", prop.second); - std::get<0>(node_n->properties_)[storage.GetPropertyIx(prop.first)] = m_prop; - auto node_m = NODE("m"); - auto n_prop = PROPERTY_LOOKUP("n", prop.second); - std::get<0>(node_m->properties_)[storage.GetPropertyIx(prop.first)] = n_prop; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node_n), PATTERN(node_m)), RETURN("n"))); - // We expect both ScanAll to come before filters (2 are joined into one), - // because they need to populate the symbol values. - CheckPlan(query, storage, ExpectScanAll(), ExpectScanAll(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchWhereBeforeExpand) { - // Test MATCH (n) -[r]- (m) WHERE n.prop < 42 RETURN n - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *as_n = NEXPR("n", IDENT("n")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), - WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN(as_n))); - // We expect Filter to come immediately after ScanAll, since it only uses `n`. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto prop = PROPERTY_PAIR("prop"); - dba.SetIndexCount(label, 1); - dba.SetIndexCount(label, prop.second, 1); - AstStorage storage; - - { - // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL RETURN n - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), - WHERE(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop)))), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect ScanAllByLabelProperty to come instead of ScanAll > Filter. - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectExpand(), ExpectProduce()); - } - - { - // Test MATCH (n :label) -[r]- (m) WHERE n.prop IS NOT NULL OR true RETURN n - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), - WHERE(OR(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop))), LITERAL(true))), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect ScanAllBy > Filter because of the "or true" condition. - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectProduce()); - } - - { - // Test MATCH (n :label) -[r]- (m) - // WHERE n.prop IS NOT NULL AND n.x = 2 RETURN n - auto prop_x = PROPERTY_PAIR("x"); - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"), EDGE("r"), NODE("m"))), - WHERE(AND(NOT(IS_NULL(PROPERTY_LOOKUP("n", prop))), EQ(PROPERTY_LOOKUP("n", prop_x), LITERAL(2)))), - RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect ScanAllByLabelProperty > Filter - // to come instead of ScanAll > Filter. - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelProperty(label, prop), ExpectFilter(), ExpectExpand(), - ExpectProduce()); - } -} - -TYPED_TEST(TestPlanner, MultiMatchWhere) { - // Test MATCH (n) -[r]- (m) MATCH (l) WHERE n.prop < 42 RETURN n - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), MATCH(PATTERN(NODE("l"))), - WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN("n"))); - // Even though WHERE is in the second MATCH clause, we expect Filter to come - // before second ScanAll, since it only uses the value from first ScanAll. - CheckPlan(query, storage, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectScanAll(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) { - // Test MATCH (n) -[r]- (m) OPTIONAL MATCH (l) WHERE n.prop < 42 RETURN n - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), OPTIONAL_MATCH(PATTERN(NODE("l"))), - WHERE(LESS(PROPERTY_LOOKUP("n", prop), LITERAL(42))), RETURN("n"))); - // Even though WHERE is in the second MATCH clause, and it uses the value from - // first ScanAll, it must remain part of the Optional. It should come before - // optional ScanAll. - std::list optional{new ExpectFilter(), new ExpectScanAll()}; - CheckPlan(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce()); - DeleteListContent(&optional); -} - -TYPED_TEST(TestPlanner, MatchReturnAsterisk) { - // Test MATCH (n) -[e]- (m) RETURN *, m.prop - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto ret = RETURN(PROPERTY_LOOKUP("m", prop), AS("m.prop")); - ret->body_.all_identifiers = true; - auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("e"), NODE("m"))), ret)); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectProduce()); - std::vector output_names; - for (const auto &output_symbol : planner.plan().OutputSymbols(symbol_table)) { - output_names.emplace_back(output_symbol.name()); - } - std::vector expected_names{"e", "m", "n", "m.prop"}; - EXPECT_EQ(output_names, expected_names); -} - -TYPED_TEST(TestPlanner, MatchReturnAsteriskSum) { - // Test MATCH (n) RETURN *, SUM(n.prop) AS s - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - AstStorage storage; - auto sum = SUM(PROPERTY_LOOKUP("n", prop)); - auto ret = RETURN(sum, AS("s")); - ret->body_.all_identifiers = true; - auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), ret)); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - auto *produce = dynamic_cast(&planner.plan()); - ASSERT_TRUE(produce); - const auto &named_expressions = produce->named_expressions_; - ASSERT_EQ(named_expressions.size(), 2); - auto *expanded_ident = dynamic_cast(named_expressions[0]->expression_); - ASSERT_TRUE(expanded_ident); - auto aggr = ExpectAggregate({sum}, {expanded_ident}); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), aggr, ExpectProduce()); - std::vector output_names; - for (const auto &output_symbol : planner.plan().OutputSymbols(symbol_table)) { - output_names.emplace_back(output_symbol.name()); - } - std::vector expected_names{"n", "s"}; - EXPECT_EQ(output_names, expected_names); -} - -TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) { - // Test UNWIND [1] AS i MERGE (n {prop: i}) - AstStorage storage; - FakeDistributedDbAccessor dba; - auto node_n = NODE("n"); - std::get<0>(node_n->properties_)[storage.GetPropertyIx("prop")] = IDENT("i"); - auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1)), AS("i")), MERGE(PATTERN(node_n)))); - std::list on_match{new ExpectScanAll(), new ExpectFilter()}; - std::list on_create{new ExpectCreateNode()}; - CheckPlan(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create)); - DeleteListContent(&on_match); - DeleteListContent(&on_create); -} - -TYPED_TEST(TestPlanner, UnwindMergeNodePropertyWithIndex) { - // Test UNWIND [1] AS i MERGE (n :label {prop: i}) with label-property index - AstStorage storage; - FakeDistributedDbAccessor dba; - const auto label_name = "label"; - const auto label = dba.Label(label_name); - const auto property = PROPERTY_PAIR("prop"); - dba.SetIndexCount(label, property.second, 1); - auto node_n = NODE("n", label_name); - std::get<0>(node_n->properties_)[storage.GetPropertyIx(property.first)] = IDENT("i"); - auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1)), AS("i")), MERGE(PATTERN(node_n)))); - std::list on_match{new ExpectScanAllByLabelPropertyValue(label, property, IDENT("i"))}; - std::list on_create{new ExpectCreateNode()}; - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectUnwind(), ExpectMerge(on_match, on_create)); - DeleteListContent(&on_match); - DeleteListContent(&on_create); -} - -TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) { - // Test OPTIONAL MATCH (n) OPTIONAL MATCH (m) RETURN n - AstStorage storage; - auto *query = - QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n"))); - std::list optional{new ExpectScanAll()}; - CheckPlan(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce()); - DeleteListContent(&optional); -} - -TYPED_TEST(TestPlanner, FunctionAggregationReturn) { - // Test RETURN sqrt(SUM(2)) AS result, 42 AS group_by - AstStorage storage; - auto sum = SUM(LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN(FN("sqrt", sum), AS("result"), group_by_literal, AS("group_by")))); - auto aggr = ExpectAggregate({sum}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, FunctionWithoutArguments) { - // Test RETURN pi() AS pi - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(RETURN(FN("pi"), AS("pi")))); - CheckPlan(query, storage, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ListLiteralAggregationReturn) { - // Test RETURN [SUM(2)] AS result, 42 AS group_by - AstStorage storage; - auto sum = SUM(LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN(LIST(sum), AS("result"), group_by_literal, AS("group_by")))); - auto aggr = ExpectAggregate({sum}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MapLiteralAggregationReturn) { - // Test RETURN {sum: SUM(2)} AS result, 42 AS group_by - AstStorage storage; - FakeDistributedDbAccessor dba; - auto sum = SUM(LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY( - SINGLE_QUERY(RETURN(MAP({storage.GetPropertyIx("sum"), sum}), AS("result"), group_by_literal, AS("group_by")))); - auto aggr = ExpectAggregate({sum}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, EmptyListIndexAggregation) { - // Test RETURN [][SUM(2)] AS result, 42 AS group_by - AstStorage storage; - auto sum = SUM(LITERAL(2)); - auto empty_list = LIST(); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN(storage.Create(empty_list, sum), - AS("result"), group_by_literal, AS("group_by")))); - // We expect to group by '42' and the empty list, because it is a - // sub-expression of a binary operator which contains an aggregation. This is - // similar to grouping by '1' in `RETURN 1 + SUM(2)`. - auto aggr = ExpectAggregate({sum}, {empty_list, group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ListSliceAggregationReturn) { - // Test RETURN [1, 2][0..SUM(2)] AS result, 42 AS group_by - AstStorage storage; - auto sum = SUM(LITERAL(2)); - auto list = LIST(LITERAL(1), LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = - QUERY(SINGLE_QUERY(RETURN(SLICE(list, LITERAL(0), sum), AS("result"), group_by_literal, AS("group_by")))); - // Similarly to EmptyListIndexAggregation test, we expect grouping by list and - // '42', because slicing is an operator. - auto aggr = ExpectAggregate({sum}, {list, group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ListWithAggregationAndGroupBy) { - // Test RETURN [sum(2), 42] - AstStorage storage; - auto sum = SUM(LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN(LIST(sum, group_by_literal), AS("result")))); - auto aggr = ExpectAggregate({sum}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, AggregatonWithListWithAggregationAndGroupBy) { - // Test RETURN sum(2), [sum(3), 42] - AstStorage storage; - auto sum2 = SUM(LITERAL(2)); - auto sum3 = SUM(LITERAL(3)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN(sum2, AS("sum2"), LIST(sum3, group_by_literal), AS("list")))); - auto aggr = ExpectAggregate({sum2, sum3}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MapWithAggregationAndGroupBy) { - // Test RETURN {lit: 42, sum: sum(2)} - AstStorage storage; - FakeDistributedDbAccessor dba; - auto sum = SUM(LITERAL(2)); - auto group_by_literal = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(RETURN( - MAP({storage.GetPropertyIx("sum"), sum}, {storage.GetPropertyIx("lit"), group_by_literal}), AS("result")))); - auto aggr = ExpectAggregate({sum}, {group_by_literal}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, AtomIndexedLabelProperty) { - // Test MATCH (n :label {property: 42, not_indexed: 0}) RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - auto not_indexed = PROPERTY_PAIR("not_indexed"); - dba.SetIndexCount(label, 1); - dba.SetIndexCount(label, property.second, 1); - auto node = NODE("n", "label"); - auto lit_42 = LITERAL(42); - std::get<0>(node->properties_)[storage.GetPropertyIx(property.first)] = lit_42; - std::get<0>(node->properties_)[storage.GetPropertyIx(not_indexed.first)] = LITERAL(0); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(node)), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, AtomPropertyWhereLabelIndexing) { - // Test MATCH (n {property: 42}) WHERE n.not_indexed AND n:label RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - auto not_indexed = PROPERTY_PAIR("not_indexed"); - dba.SetIndexCount(label, property.second, 0); - auto node = NODE("n"); - auto lit_42 = LITERAL(42); - std::get<0>(node->properties_)[storage.GetPropertyIx(property.first)] = lit_42; - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(node)), - WHERE(AND(PROPERTY_LOOKUP("n", not_indexed), - storage.Create( - IDENT("n"), std::vector{storage.GetLabelIx("label")}))), - RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, WhereIndexedLabelProperty) { - // Test MATCH (n :label) WHERE n.property = 42 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - dba.SetIndexCount(label, property.second, 0); - auto lit_42 = LITERAL(42); - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(EQ(PROPERTY_LOOKUP("n", property), lit_42)), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, BestPropertyIndexed) { - // Test MATCH (n :label) WHERE n.property = 1 AND n.better = 42 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = dba.Property("property"); - // Add a vertex with :label+property combination, so that the best - // :label+better remains empty and thus better choice. - dba.SetIndexCount(label, property, 1); - auto better = PROPERTY_PAIR("better"); - dba.SetIndexCount(label, better.second, 0); - auto lit_42 = LITERAL(42); - auto *query = QUERY( - SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(AND(EQ(PROPERTY_LOOKUP("n", property), LITERAL(1)), EQ(PROPERTY_LOOKUP("n", better), lit_42))), - RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, better, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MultiPropertyIndexScan) { - // Test MATCH (n :label1), (m :label2) WHERE n.prop1 = 1 AND m.prop2 = 2 - // RETURN n, m - FakeDistributedDbAccessor dba; - auto label1 = dba.Label("label1"); - auto label2 = dba.Label("label2"); - auto prop1 = PROPERTY_PAIR("prop1"); - auto prop2 = PROPERTY_PAIR("prop2"); - dba.SetIndexCount(label1, prop1.second, 0); - dba.SetIndexCount(label2, prop2.second, 0); - AstStorage storage; - auto lit_1 = LITERAL(1); - auto lit_2 = LITERAL(2); - auto *query = QUERY(SINGLE_QUERY( - MATCH(PATTERN(NODE("n", "label1")), PATTERN(NODE("m", "label2"))), - WHERE(AND(EQ(PROPERTY_LOOKUP("n", prop1), lit_1), EQ(PROPERTY_LOOKUP("m", prop2), lit_2))), RETURN("n", "m"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label1, prop1, lit_1), - ExpectScanAllByLabelPropertyValue(label2, prop2, lit_2), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, WhereIndexedLabelPropertyRange) { - // Test MATCH (n :label) WHERE n.property REL_OP 42 RETURN n - // REL_OP is one of: `<`, `<=`, `>`, `>=` - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = dba.Property("property"); - dba.SetIndexCount(label, property, 0); - AstStorage storage; - auto lit_42 = LITERAL(42); - auto n_prop = PROPERTY_LOOKUP("n", property); - auto check_planned_range = [&label, &property, &dba](const auto &rel_expr, auto lower_bound, auto upper_bound) { - // Shadow the first storage, so that the query is created in this one. - AstStorage storage; - storage.GetLabelIx("label"); - storage.GetPropertyIx("property"); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(rel_expr), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, - ExpectScanAllByLabelPropertyRange(label, property, lower_bound, upper_bound), ExpectProduce()); - }; - { - // Test relation operators which form an upper bound for range. - std::vector> upper_bound_rel_op{ - std::make_pair(LESS(n_prop, lit_42), Bound::Type::EXCLUSIVE), - std::make_pair(LESS_EQ(n_prop, lit_42), Bound::Type::INCLUSIVE), - std::make_pair(GREATER(lit_42, n_prop), Bound::Type::EXCLUSIVE), - std::make_pair(GREATER_EQ(lit_42, n_prop), Bound::Type::INCLUSIVE)}; - for (const auto &rel_op : upper_bound_rel_op) { - check_planned_range(rel_op.first, std::nullopt, Bound(lit_42, rel_op.second)); - } - } - { - // Test relation operators which form a lower bound for range. - std::vector> lower_bound_rel_op{ - std::make_pair(LESS(lit_42, n_prop), Bound::Type::EXCLUSIVE), - std::make_pair(LESS_EQ(lit_42, n_prop), Bound::Type::INCLUSIVE), - std::make_pair(GREATER(n_prop, lit_42), Bound::Type::EXCLUSIVE), - std::make_pair(GREATER_EQ(n_prop, lit_42), Bound::Type::INCLUSIVE)}; - for (const auto &rel_op : lower_bound_rel_op) { - check_planned_range(rel_op.first, Bound(lit_42, rel_op.second), std::nullopt); - } - } -} - -TYPED_TEST(TestPlanner, WherePreferEqualityIndexOverRange) { - // Test MATCH (n :label) WHERE n.property = 42 AND n.property > 0 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - dba.SetIndexCount(label, property.second, 0); - auto lit_42 = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY( - MATCH(PATTERN(NODE("n", "label"))), - WHERE(AND(EQ(PROPERTY_LOOKUP("n", property), lit_42), GREATER(PROPERTY_LOOKUP("n", property), LITERAL(0)))), - RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, property, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, UnableToUsePropertyIndex) { - // Test MATCH (n: label) WHERE n.property = n.property RETURN n - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = dba.Property("property"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, property, 0); - AstStorage storage; - auto *query = - QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(EQ(PROPERTY_LOOKUP("n", property), PROPERTY_LOOKUP("n", property))), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We can only get ScanAllByLabelIndex, because we are comparing properties - // with those on the same node. - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, SecondPropertyIndex) { - // Test MATCH (n :label), (m :label) WHERE m.property = n.property RETURN n - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, dba.Property("property"), 0); - AstStorage storage; - auto n_prop = PROPERTY_LOOKUP("n", property); - auto m_prop = PROPERTY_LOOKUP("m", property); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label")), PATTERN(NODE("m", "label"))), - WHERE(EQ(m_prop, n_prop)), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), - // Note: We are scanning for m, therefore property should equal n_prop. - ExpectScanAllByLabelPropertyValue(label, property, n_prop), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ReturnSumGroupByAll) { - // Test RETURN sum([1,2,3]), all(x in [1] where x = 1) - AstStorage storage; - auto sum = SUM(LIST(LITERAL(1), LITERAL(2), LITERAL(3))); - auto *all = ALL("x", LIST(LITERAL(1)), WHERE(EQ(IDENT("x"), LITERAL(1)))); - auto *query = QUERY(SINGLE_QUERY(RETURN(sum, AS("sum"), all, AS("all")))); - auto aggr = ExpectAggregate({sum}, {all}); - CheckPlan(query, storage, aggr, ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchExpandVariable) { - // Test MATCH (n) -[r *..3]-> (m) RETURN r - AstStorage storage; - auto edge = EDGE_VARIABLE("r"); - edge->upper_bound_ = LITERAL(3); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchExpandVariableNoBounds) { - // Test MATCH (n) -[r *]-> (m) RETURN r - AstStorage storage; - auto edge = EDGE_VARIABLE("r"); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchExpandVariableInlinedFilter) { - // Test MATCH (n) -[r :type * {prop: 42}]-> (m) RETURN r - FakeDistributedDbAccessor dba; - auto type = "type"; - auto prop = PROPERTY_PAIR("prop"); - AstStorage storage; - auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH, {type}); - std::get<0>(edge->properties_)[storage.GetPropertyIx(prop.first)] = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), - ExpectExpandVariable(), // Filter is both inlined and post-expand - ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchExpandVariableNotInlinedFilter) { - // Test MATCH (n) -[r :type * {prop: m.prop}]-> (m) RETURN r - FakeDistributedDbAccessor dba; - auto type = "type"; - auto prop = PROPERTY_PAIR("prop"); - AstStorage storage; - auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH, {type}); - std::get<0>(edge->properties_)[storage.GetPropertyIx(prop.first)] = EQ(PROPERTY_LOOKUP("m", prop), LITERAL(42)); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), ExpectExpandVariable(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchExpandVariableTotalWeightSymbol) { - // Test MATCH p = (a {id: 0})-[r* wShortest (e, v | 1) total_weight]->(b) - // RETURN * - FakeDistributedDbAccessor dba; - AstStorage storage; - - auto edge = EDGE_VARIABLE("r", Type::WEIGHTED_SHORTEST_PATH, Direction::BOTH, {}, nullptr, nullptr, nullptr, nullptr, - nullptr, IDENT("total_weight")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("*"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - auto *root = dynamic_cast(&planner.plan()); - - ASSERT_TRUE(root); - - const auto &nes = root->named_expressions_; - EXPECT_TRUE(nes.size() == 4); - - std::vector names(nes.size()); - std::transform(nes.begin(), nes.end(), names.begin(), [](const auto *ne) { return ne->name_; }); - - EXPECT_TRUE(root->named_expressions_.size() == 4); - EXPECT_TRUE(memgraph::utils::Contains(names, "m")); - EXPECT_TRUE(memgraph::utils::Contains(names, "n")); - EXPECT_TRUE(memgraph::utils::Contains(names, "r")); - EXPECT_TRUE(memgraph::utils::Contains(names, "total_weight")); -} - -TYPED_TEST(TestPlanner, UnwindMatchVariable) { - // Test UNWIND [1,2,3] AS depth MATCH (n) -[r*d]-> (m) RETURN r - AstStorage storage; - auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::OUT); - edge->lower_bound_ = IDENT("d"); - edge->upper_bound_ = IDENT("d"); - auto *query = QUERY(SINGLE_QUERY(UNWIND(LIST(LITERAL(1), LITERAL(2), LITERAL(3)), AS("d")), - MATCH(PATTERN(NODE("n"), edge, NODE("m"))), RETURN("r"))); - CheckPlan(query, storage, ExpectUnwind(), ExpectScanAll(), ExpectExpandVariable(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchBfs) { - // Test MATCH (n) -[r:type *..10 (r, n|n)]-> (m) RETURN r - FakeDistributedDbAccessor dba; - AstStorage storage; - auto edge_type = storage.GetEdgeTypeIx("type"); - auto *bfs = - storage.Create(IDENT("r"), memgraph::query::EdgeAtom::Type::BREADTH_FIRST, - Direction::OUT, std::vector{edge_type}); - bfs->filter_lambda_.inner_edge = IDENT("r"); - bfs->filter_lambda_.inner_node = IDENT("n"); - bfs->filter_lambda_.expression = IDENT("n"); - bfs->upper_bound_ = LITERAL(10); - auto *as_r = NEXPR("r", IDENT("r")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), bfs, NODE("m"))), RETURN(as_r))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpandBfs(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchDoubleScanToExpandExisting) { - // Test MATCH (n) -[r]- (m :label) RETURN r - FakeDistributedDbAccessor dba; - auto label = "label"; - dba.SetIndexCount(dba.Label(label), 0); - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m", label))), RETURN("r"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect 2x ScanAll and then Expand, since we are guessing that is - // faster (due to low label index vertex count). - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectScanAllByLabel(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchScanToExpand) { - // Test MATCH (n) -[r]- (m :label {property: 1}) RETURN r - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = dba.Property("property"); - // Fill vertices to the max + 1. - dba.SetIndexCount(label, property, FLAGS_query_vertex_count_to_expand_existing + 1); - dba.SetIndexCount(label, FLAGS_query_vertex_count_to_expand_existing + 1); - AstStorage storage; - auto node_m = NODE("m", "label"); - std::get<0>(node_m->properties_)[storage.GetPropertyIx("property")] = LITERAL(1); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), node_m)), RETURN("r"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - // We expect 1x ScanAll and then Expand, since we are guessing that - // is faster (due to high label index vertex count). - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectExpand(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, MatchWhereAndSplit) { - // Test MATCH (n) -[r]- (m) WHERE n.prop AND r.prop RETURN m - FakeDistributedDbAccessor dba; - auto prop = PROPERTY_PAIR("prop"); - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), - WHERE(AND(PROPERTY_LOOKUP("n", prop), PROPERTY_LOOKUP("r", prop))), RETURN("m"))); - // We expect `n.prop` filter right after scanning `n`. - CheckPlan(query, storage, ExpectScanAll(), ExpectFilter(), ExpectExpand(), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ReturnAsteriskOmitsLambdaSymbols) { - // Test MATCH (n) -[r* (ie, in | true)]- (m) RETURN * - AstStorage storage; - auto edge = EDGE_VARIABLE("r", Type::DEPTH_FIRST, Direction::BOTH); - edge->filter_lambda_.inner_edge = IDENT("ie"); - edge->filter_lambda_.inner_node = IDENT("in"); - edge->filter_lambda_.expression = LITERAL(true); - auto ret = storage.Create(); - ret->body_.all_identifiers = true; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), edge, NODE("m"))), ret)); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - auto *produce = dynamic_cast(&planner.plan()); - ASSERT_TRUE(produce); - std::vector outputs; - for (const auto &output_symbol : produce->OutputSymbols(symbol_table)) { - outputs.emplace_back(output_symbol.name()); - } - // We expect `*` expanded to `n`, `r` and `m`. - EXPECT_EQ(outputs.size(), 3); - for (const auto &name : {"n", "r", "m"}) { - EXPECT_TRUE(memgraph::utils::Contains(outputs, name)); - } -} - -TYPED_TEST(TestPlanner, FilterRegexMatchIndex) { - // Test MATCH (n :label) WHERE n.prop =~ "regex" RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - auto label = dba.Label("label"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, prop, 0); - auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), WHERE(regex_match), RETURN("n"))); - // We expect that we use index by property range where lower bound is an empty - // string. Filter must still remain in place, because we don't have regex - // based index. - Bound lower_bound(LITERAL(""), Bound::Type::INCLUSIVE); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyRange(label, prop, lower_bound, std::nullopt), - ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, FilterRegexMatchPreferEqualityIndex) { - // Test MATCH (n :label) WHERE n.prop =~ "regex" AND n.prop = 42 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = PROPERTY_PAIR("prop"); - auto label = dba.Label("label"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, prop.second, 0); - auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); - auto *lit_42 = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(AND(regex_match, EQ(PROPERTY_LOOKUP("n", prop), lit_42))), RETURN("n"))); - // We expect that we use index by property value equal to 42, because that's - // much better than property range for regex matching. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, prop, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, FilterRegexMatchPreferEqualityIndex2) { - // Test MATCH (n :label) - // WHERE n.prop =~ "regex" AND n.prop = 42 AND n.prop > 0 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = PROPERTY_PAIR("prop"); - auto label = dba.Label("label"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, prop.second, 0); - auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); - auto *lit_42 = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(AND(AND(regex_match, EQ(PROPERTY_LOOKUP("n", prop), lit_42)), - GREATER(PROPERTY_LOOKUP("n", prop), LITERAL(0)))), - RETURN("n"))); - // We expect that we use index by property value equal to 42, because that's - // much better than property range. - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyValue(label, prop, lit_42), ExpectFilter(), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, FilterRegexMatchPreferRangeIndex) { - // Test MATCH (n :label) WHERE n.prop =~ "regex" AND n.prop > 42 RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto prop = dba.Property("prop"); - auto label = dba.Label("label"); - dba.SetIndexCount(label, 0); - dba.SetIndexCount(label, prop, 0); - auto *regex_match = storage.Create(PROPERTY_LOOKUP("n", prop), LITERAL("regex")); - auto *lit_42 = LITERAL(42); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(AND(regex_match, GREATER(PROPERTY_LOOKUP("n", prop), lit_42))), RETURN("n"))); - // We expect that we use index by property range on a concrete value (42), as - // it is much better than using a range from empty string for regex matching. - Bound lower_bound(lit_42, Bound::Type::EXCLUSIVE); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabelPropertyRange(label, prop, lower_bound, std::nullopt), - ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CallProcedureStandalone) { - // Test CALL proc(1,2,3) YIELD field AS result - AstStorage storage; - auto *ast_call = storage.Create(); - ast_call->procedure_name_ = "proc"; - ast_call->arguments_ = {LITERAL(1), LITERAL(2), LITERAL(3)}; - ast_call->result_fields_ = {"field"}; - ast_call->result_identifiers_ = {IDENT("result")}; - auto *query = QUERY(SINGLE_QUERY(ast_call)); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - std::vector result_syms; - result_syms.reserve(ast_call->result_identifiers_.size()); - for (const auto *ident : ast_call->result_identifiers_) { - result_syms.push_back(symbol_table.at(*ident)); - } - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan( - planner.plan(), symbol_table, - ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms)); -} - -TYPED_TEST(TestPlanner, CallProcedureAfterScanAll) { - // Test MATCH (n) CALL proc(n) YIELD field AS result RETURN result - AstStorage storage; - auto *ast_call = storage.Create(); - ast_call->procedure_name_ = "proc"; - ast_call->arguments_ = {IDENT("n")}; - ast_call->result_fields_ = {"field"}; - ast_call->result_identifiers_ = {IDENT("result")}; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), ast_call, RETURN("result"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - std::vector result_syms; - result_syms.reserve(ast_call->result_identifiers_.size()); - for (const auto *ident : ast_call->result_identifiers_) { - result_syms.push_back(symbol_table.at(*ident)); - } - FakeDistributedDbAccessor dba; - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), - ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms), - ExpectProduce()); -} - -TYPED_TEST(TestPlanner, CallProcedureBeforeScanAll) { - // Test CALL proc() YIELD field MATCH (n) WHERE n.prop = field RETURN n - AstStorage storage; - auto *ast_call = storage.Create(); - ast_call->procedure_name_ = "proc"; - ast_call->result_fields_ = {"field"}; - ast_call->result_identifiers_ = {IDENT("field")}; - FakeDistributedDbAccessor dba; - auto property = dba.Property("prop"); - auto *query = QUERY(SINGLE_QUERY(ast_call, MATCH(PATTERN(NODE("n"))), - WHERE(EQ(PROPERTY_LOOKUP("n", property), IDENT("field"))), RETURN("n"))); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - std::vector result_syms; - result_syms.reserve(ast_call->result_identifiers_.size()); - for (const auto *ident : ast_call->result_identifiers_) { - result_syms.push_back(symbol_table.at(*ident)); - } - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, - ExpectCallProcedure(ast_call->procedure_name_, ast_call->arguments_, ast_call->result_fields_, result_syms), - ExpectScanAll(), ExpectFilter(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ScanAllById) { - // Test MATCH (n) WHERE id(n) = 42 RETURN n - AstStorage storage; - auto *query = - QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), WHERE(EQ(FN("id", IDENT("n")), LITERAL(42))), RETURN("n"))); - CheckPlan(query, storage, ExpectScanAllById(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, ScanAllByIdExpandToExisting) { - // Test MATCH (n)-[r]-(m) WHERE id(m) = 42 RETURN r - AstStorage storage; - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), - WHERE(EQ(FN("id", IDENT("m")), LITERAL(42))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), ExpectScanAllById(), ExpectExpand(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, BfsToExisting) { - // Test MATCH (n)-[r *bfs]-(m) WHERE id(m) = 42 RETURN r - AstStorage storage; - auto *bfs = storage.Create(IDENT("r"), memgraph::query::EdgeAtom::Type::BREADTH_FIRST, - Direction::BOTH); - bfs->filter_lambda_.inner_edge = IDENT("ie"); - bfs->filter_lambda_.inner_node = IDENT("in"); - bfs->filter_lambda_.expression = LITERAL(true); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), bfs, NODE("m"))), - WHERE(EQ(FN("id", IDENT("m")), LITERAL(42))), RETURN("r"))); - CheckPlan(query, storage, ExpectScanAll(), ExpectScanAllById(), ExpectExpandBfs(), ExpectProduce()); -} - -TYPED_TEST(TestPlanner, LabelPropertyInListValidOptimization) { - // Test MATCH (n:label) WHERE n.property IN ['a'] RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - auto *lit_list_a = LIST(LITERAL('a')); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(IN_LIST(PROPERTY_LOOKUP("n", property), lit_list_a)), RETURN("n"))); - { - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); - } - { - dba.SetIndexCount(label, 1); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); - } - { - dba.SetIndexCount(label, property.second, 1); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectUnwind(), - ExpectScanAllByLabelPropertyValue(label, property, lit_list_a), ExpectProduce()); - } -} - -TYPED_TEST(TestPlanner, LabelPropertyInListWhereLabelPropertyOnLeftNotListOnRight) { - // Test MATCH (n:label) WHERE n.property IN 'a' RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - auto *lit_a = LITERAL('a'); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(IN_LIST(PROPERTY_LOOKUP("n", property), lit_a)), RETURN("n"))); - { - dba.SetIndexCount(label, property.second, 1); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); - } -} - -TYPED_TEST(TestPlanner, LabelPropertyInListWhereLabelPropertyOnRight) { - // Test MATCH (n:label) WHERE ['a'] IN n.property RETURN n - AstStorage storage; - FakeDistributedDbAccessor dba; - auto label = dba.Label("label"); - auto property = PROPERTY_PAIR("property"); - auto *lit_list_a = LIST(LITERAL('a')); - auto *query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n", "label"))), - WHERE(IN_LIST(lit_list_a, PROPERTY_LOOKUP("n", property))), RETURN("n"))); - { - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectFilter(), ExpectProduce()); - } - { - dba.SetIndexCount(label, 1); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); - } - { - dba.SetIndexCount(label, property.second, 1); - auto symbol_table = memgraph::query::MakeSymbolTable(query); - auto planner = MakePlanner(&dba, storage, symbol_table, query); - CheckPlan(planner.plan(), symbol_table, ExpectScanAllByLabel(), ExpectFilter(), ExpectProduce()); - } -} - -TYPED_TEST(TestPlanner, Foreach) { - AstStorage storage; - FakeDistributedDbAccessor dba; - { - auto *i = NEXPR("i", IDENT("i")); - auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {CREATE(PATTERN(NODE("n")))}))); - auto create = ExpectCreateNode(); - std::list updates{&create}; - std::list input; - CheckPlan(query, storage, ExpectForeach(input, updates)); - } - { - auto *i = NEXPR("i", IDENT("i")); - auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {DELETE(IDENT("i"))}))); - auto del = ExpectDelete(); - std::list updates{&del}; - std::list input; - CheckPlan(query, storage, ExpectForeach({input}, updates)); - } - { - auto prop = dba.Property("prop"); - auto *i = NEXPR("i", IDENT("i")); - auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {SET(PROPERTY_LOOKUP("i", prop), LITERAL(10))}))); - auto set_prop = ExpectSetProperty(); - std::list updates{&set_prop}; - std::list input; - CheckPlan(query, storage, ExpectForeach({input}, updates)); - } - { - auto *i = NEXPR("i", IDENT("i")); - auto *j = NEXPR("j", IDENT("j")); - auto *query = QUERY(SINGLE_QUERY(FOREACH(i, {FOREACH(j, {CREATE(PATTERN(NODE("n"))), DELETE(IDENT("i"))})}))); - auto create = ExpectCreateNode(); - auto del = ExpectDelete(); - std::list input; - std::list nested_updates{{&create, &del}}; - auto nested_foreach = ExpectForeach(input, nested_updates); - std::list updates{&nested_foreach}; - CheckPlan(query, storage, ExpectForeach(input, updates)); - } - { - auto *i = NEXPR("i", IDENT("i")); - auto *j = NEXPR("j", IDENT("j")); - auto create = ExpectCreateNode(); - std::list empty; - std::list updates{&create}; - auto input_op = ExpectForeach(empty, updates); - std::list input{&input_op}; - auto *query = - QUERY(SINGLE_QUERY(FOREACH(i, {CREATE(PATTERN(NODE("n")))}), FOREACH(j, {CREATE(PATTERN(NODE("n")))}))); - CheckPlan(query, storage, ExpectForeach(input, updates)); - } -} -*/ } // namespace From 3b06db5a02181d4b16495062b447f8779a403ebb Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 16:13:27 +0100 Subject: [PATCH 033/251] Clang-tidy --- src/query/v2/plan/operator.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index b545b742f..e55859cca 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -589,9 +589,12 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { SCOPED_PROFILE_OP(op_name_); if (!own_multi_frames_.has_value()) { - // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) + // NOLINTBEGIN(cppcoreguidelines-narrowing-conversions) + // NOLINTBEGIN(bugprone-narrowing-conversions) own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); + // NOLINTEND(bugprone-narrowing-conversions) + // NOLINTEND(cppcoreguidelines-narrowing-conversions) PrepareNextFrames(context); } From a97d9945153a2af39e076ded76ade23c23ef92eb Mon Sep 17 00:00:00 2001 From: jbajic Date: Thu, 12 Jan 2023 16:18:16 +0100 Subject: [PATCH 034/251] Fix vertice split --- src/storage/v3/delta.hpp | 5 +- src/storage/v3/shard.cpp | 6 +- tests/unit/CMakeLists.txt | 3 + tests/unit/storage_v3_shard_split.cpp | 87 +++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 tests/unit/storage_v3_shard_split.cpp diff --git a/src/storage/v3/delta.hpp b/src/storage/v3/delta.hpp index 69da63e77..39c9975f6 100644 --- a/src/storage/v3/delta.hpp +++ b/src/storage/v3/delta.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -133,7 +133,8 @@ inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer struct Delta { // Needed for splits - boost::uuids::uuid uuid; + // TODO Replace this with int identifier + boost::uuids::uuid uuid{boost::uuids::uuid()}; enum class Action { // Used for both Vertex and Edge diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 856906385..bf89ed58c 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1066,12 +1066,14 @@ void Shard::ScanDeltas(std::set &collected_transactions_start_id, Delt VertexContainer Shard::CollectVertices(std::set &collected_transactions_start_id, const PrimaryKey &split_key) { VertexContainer splitted_data; - auto split_key_it = vertices_.find(split_key); - for (; split_key_it != vertices_.end(); split_key_it++) { + auto split_key_it = vertices_.find(split_key); + while (split_key_it != vertices_.end()) { // Go through deltas and pick up transactions start_id ScanDeltas(collected_transactions_start_id, split_key_it->second.delta); + auto next_it = std::next(split_key_it); splitted_data.insert(vertices_.extract(split_key_it->first)); + split_key_it = next_it; } return splitted_data; } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 5bfa26afd..cc23a3e5e 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -291,6 +291,9 @@ target_link_libraries(${test_prefix}storage_v3_expr mg-storage-v3 mg-expr) add_unit_test(storage_v3_schema.cpp) target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3) +add_unit_test(storage_v3_shard_split.cpp) +target_link_libraries(${test_prefix}storage_v3_shard_split mg-storage-v3 mg-query-v2) + # Test mg-query-v2 # These are commented out because of the new TypedValue in the query engine # add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp) diff --git a/tests/unit/storage_v3_shard_split.cpp b/tests/unit/storage_v3_shard_split.cpp new file mode 100644 index 000000000..d1bafb65c --- /dev/null +++ b/tests/unit/storage_v3_shard_split.cpp @@ -0,0 +1,87 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include + +#include "query/v2/requests.hpp" +#include "storage/v3/id_types.hpp" +#include "storage/v3/key_store.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/shard.hpp" +#include "storage/v3/vertex_id.hpp" + +namespace memgraph::storage::v3::tests { + +class ShardSplitTest : public testing::Test { + protected: + void SetUp() override { storage.StoreMapping({{1, "label"}, {2, "property"}, {3, "edge_property"}}); } + + const PropertyId primary_property{PropertyId::FromUint(2)}; + std::vector schema_property_vector = { + storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}}; + const std::vector min_pk{PropertyValue{0}}; + const LabelId primary_label{LabelId::FromUint(1)}; + const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)}; + Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector}; + + coordinator::Hlc last_hlc{0, io::Time{}}; + + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += std::chrono::seconds(1); + return last_hlc; + } +}; + +TEST_F(ShardSplitTest, TestBasicSplitWithVertices) { + auto acc = storage.Access(GetNextHlc()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError()); + acc.Commit(GetNextHlc()); + storage.CollectGarbage(GetNextHlc().coordinator_wall_clock); + + auto splitted_data = storage.PerformSplit({PropertyValue(4)}); + EXPECT_EQ(splitted_data.vertices.size(), 3); + EXPECT_EQ(splitted_data.edges->size(), 0); + EXPECT_EQ(splitted_data.transactions.size(), 0); +} + +TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) { + auto acc = storage.Access(GetNextHlc()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError()); + + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(0)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(0)) + .HasError()); + + acc.Commit(GetNextHlc()); + storage.CollectGarbage(GetNextHlc().coordinator_wall_clock); + + auto splitted_data = storage.PerformSplit({PropertyValue(4)}); +} + +} // namespace memgraph::storage::v3::tests From 791972a6b841d7f77df6c7e37948037950195692 Mon Sep 17 00:00:00 2001 From: jbajic Date: Thu, 12 Jan 2023 16:21:46 +0100 Subject: [PATCH 035/251] Fix edge split test --- tests/unit/storage_v3_shard_split.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/storage_v3_shard_split.cpp b/tests/unit/storage_v3_shard_split.cpp index d1bafb65c..716734433 100644 --- a/tests/unit/storage_v3_shard_split.cpp +++ b/tests/unit/storage_v3_shard_split.cpp @@ -72,16 +72,19 @@ TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) { VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0)) .HasError()); EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, - VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(0)) + VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1)) .HasError()); EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}}, - VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(0)) + VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2)) .HasError()); acc.Commit(GetNextHlc()); storage.CollectGarbage(GetNextHlc().coordinator_wall_clock); auto splitted_data = storage.PerformSplit({PropertyValue(4)}); + EXPECT_EQ(splitted_data.vertices.size(), 3); + EXPECT_EQ(splitted_data.edges->size(), 2); + EXPECT_EQ(splitted_data.transactions.size(), 0); } } // namespace memgraph::storage::v3::tests From 61d84bd62253948d1dae6982d78b904267b44304 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 16:58:09 +0100 Subject: [PATCH 036/251] Set the type of the frame size to size_t from int64_t --- src/expr/interpret/frame.hpp | 10 +++++----- src/query/v2/multiframe.cpp | 4 ++-- src/query/v2/multiframe.hpp | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/expr/interpret/frame.hpp b/src/expr/interpret/frame.hpp index 1cd6a99ce..72bfcf245 100644 --- a/src/expr/interpret/frame.hpp +++ b/src/expr/interpret/frame.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -23,9 +23,9 @@ namespace memgraph::expr { class Frame { public: /// Create a Frame of given size backed by a utils::NewDeleteResource() - explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); } + explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); } - Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); } + Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); } TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; } const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; } @@ -43,9 +43,9 @@ class Frame { class FrameWithValidity final : public Frame { public: - explicit FrameWithValidity(int64_t size) : Frame(size), is_valid_(false) {} + explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {} - FrameWithValidity(int64_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {} + FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {} bool IsValid() const noexcept { return is_valid_; } void MakeValid() noexcept { is_valid_ = true; } diff --git a/src/query/v2/multiframe.cpp b/src/query/v2/multiframe.cpp index 02d396fda..835cdbc0f 100644 --- a/src/query/v2/multiframe.cpp +++ b/src/query/v2/multiframe.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -24,7 +24,7 @@ static_assert(std::forward_iterator); static_assert(std::forward_iterator); static_assert(std::forward_iterator); -MultiFrame::MultiFrame(int64_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory) +MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory) : frames_(utils::pmr::vector( number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) { MG_ASSERT(number_of_frames > 0); diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index 49dad940a..28c972ac4 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -30,7 +30,7 @@ class MultiFrame { friend class ValidFramesReader; friend class InvalidFramesPopulator; - MultiFrame(int64_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory); + MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory); ~MultiFrame() = default; MultiFrame(const MultiFrame &other); From db13ee96b6ee8b33d9bef0026f6bdf5ee9933c11 Mon Sep 17 00:00:00 2001 From: jbajic Date: Thu, 12 Jan 2023 17:02:27 +0100 Subject: [PATCH 037/251] Fix transaction split --- src/storage/v3/shard.cpp | 11 +++++++---- src/storage/v3/transaction.hpp | 9 +++++---- tests/unit/storage_v3_shard_split.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index bf89ed58c..803678c8a 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1058,7 +1058,7 @@ std::optional Shard::ShouldSplit() const noexcept { void Shard::ScanDeltas(std::set &collected_transactions_start_id, Delta *delta) const { while (delta != nullptr) { - collected_transactions_start_id.insert(delta->command_id); + collected_transactions_start_id.insert(delta->commit_info->start_or_commit_timestamp.logical_id); delta = delta->next; } } @@ -1126,8 +1126,9 @@ void Shard::AlignClonedTransaction(Transaction &cloned_transaction, const Transa auto *cloned_delta = &*cloned_delta_it; while (delta != nullptr) { // Align delta - cloned_delta->next = &*std::ranges::find_if(cloned_transactions.at(delta->command_id).deltas, - [delta](const auto &elem) { return elem.uuid == delta->uuid; }); + cloned_delta->next = &*std::ranges::find_if( + cloned_transactions.at(delta->commit_info->start_or_commit_timestamp.logical_id).deltas, + [delta](const auto &elem) { return elem.uuid == delta->uuid; }); // Align prev ptr auto ptr = delta->prev.Get(); switch (ptr.type) { @@ -1140,6 +1141,8 @@ void Shard::AlignClonedTransaction(Transaction &cloned_transaction, const Transa break; } case PreviousPtr::Type::VERTEX: { + // What if the vertex is already moved to garbage collection... + // Make test when you have deleted vertex auto *cloned_vertex = &*cloned_vertices.find(ptr.vertex->first); cloned_delta->prev.Set(cloned_vertex); break; @@ -1176,7 +1179,7 @@ std::map Shard::CollectTransactions(const std::set transactions; for (const auto commit_start : collected_transactions_start_id) { - transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()}); + transactions.insert({commit_start, start_logical_id_to_transaction_.at(commit_start)->Clone()}); } // It is necessary to clone all the transactions first so we have new addresses // for deltas, before doing alignment of deltas and prev_ptr diff --git a/src/storage/v3/transaction.hpp b/src/storage/v3/transaction.hpp index dba65067e..6c396d969 100644 --- a/src/storage/v3/transaction.hpp +++ b/src/storage/v3/transaction.hpp @@ -31,12 +31,12 @@ struct CommitInfo { }; struct Transaction { - Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, uint64_t command_id, bool must_abort, - bool is_aborted, IsolationLevel isolation_level) + Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, std::list deltas, + uint64_t command_id, bool must_abort, bool is_aborted, IsolationLevel isolation_level) : start_timestamp{start_timestamp}, commit_info{std::make_unique(new_commit_info)}, command_id(command_id), - deltas(CopyDeltas(commit_info.get())), + deltas(std::move(deltas)), must_abort(must_abort), is_aborted(is_aborted), isolation_level(isolation_level){}; @@ -108,7 +108,8 @@ struct Transaction { // This does not solve the whole problem of copying deltas Transaction Clone() const { - return {start_timestamp, *commit_info, command_id, must_abort, is_aborted, isolation_level}; + return {start_timestamp, *commit_info, CopyDeltas(commit_info.get()), command_id, must_abort, + is_aborted, isolation_level}; } coordinator::Hlc start_timestamp; diff --git a/tests/unit/storage_v3_shard_split.cpp b/tests/unit/storage_v3_shard_split.cpp index 716734433..ebfca2921 100644 --- a/tests/unit/storage_v3_shard_split.cpp +++ b/tests/unit/storage_v3_shard_split.cpp @@ -87,4 +87,29 @@ TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) { EXPECT_EQ(splitted_data.transactions.size(), 0); } +TEST_F(ShardSplitTest, TestBasicSplit) { + auto acc = storage.Access(GetNextHlc()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError()); + + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2)) + .HasError()); + + auto splitted_data = storage.PerformSplit({PropertyValue(4)}); + EXPECT_EQ(splitted_data.vertices.size(), 3); + EXPECT_EQ(splitted_data.edges->size(), 2); + EXPECT_EQ(splitted_data.transactions.size(), 1); +} + } // namespace memgraph::storage::v3::tests From 4c25a4dfbd8f6b902b48fc15189b61aee0e63318 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 20:17:35 +0100 Subject: [PATCH 038/251] Remove unnecessary comments --- src/query/v2/plan/operator.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index e55859cca..e9c8652a2 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -589,12 +589,8 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { SCOPED_PROFILE_OP(op_name_); if (!own_multi_frames_.has_value()) { - // NOLINTBEGIN(cppcoreguidelines-narrowing-conversions) - // NOLINTBEGIN(bugprone-narrowing-conversions) own_multi_frames_.emplace(MultiFrame(input_multi_frame.GetFirstFrame().elems().size(), kNumberOfFramesInMultiframe, input_multi_frame.GetMemoryResource())); - // NOLINTEND(bugprone-narrowing-conversions) - // NOLINTEND(cppcoreguidelines-narrowing-conversions) PrepareNextFrames(context); } From d22c962af405d9aafc26320399de9782824f1ecf Mon Sep 17 00:00:00 2001 From: gvolfing Date: Thu, 12 Jan 2023 21:25:40 +0100 Subject: [PATCH 039/251] Turn RequestRouter into an interface again --- src/query/v2/request_router.hpp | 4 +--- tests/unit/query_v2_expression_evaluator.cpp | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/v2/request_router.hpp b/src/query/v2/request_router.hpp index 934190cb4..6004c0b88 100644 --- a/src/query/v2/request_router.hpp +++ b/src/query/v2/request_router.hpp @@ -116,9 +116,7 @@ class RequestRouterInterface { virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0; virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0; // TODO - (gvolfing) Implement this function in the mocked class. - virtual std::vector GetSchemaForLabel(storage::v3::LabelId /*label*/) const { - return std::vector{}; - }; + virtual std::vector GetSchemaForLabel(storage::v3::LabelId /*label*/) const = 0; }; // TODO(kostasrim)rename this class template diff --git a/tests/unit/query_v2_expression_evaluator.cpp b/tests/unit/query_v2_expression_evaluator.cpp index fb875a0f2..098a94981 100644 --- a/tests/unit/query_v2_expression_evaluator.cpp +++ b/tests/unit/query_v2_expression_evaluator.cpp @@ -128,6 +128,10 @@ class MockedRequestRouter : public RequestRouterInterface { bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override { return true; } + std::vector GetSchemaForLabel(storage::v3::LabelId /*label*/) const override { + return std::vector{}; + }; + private: void SetUpNameIdMappers() { std::unordered_map id_to_name; From 668f7857b1fb84eaebbfa626554c799a2dbc0541 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Fri, 13 Jan 2023 12:38:50 +0100 Subject: [PATCH 040/251] Pass the correct expression to the operator --- src/query/v2/plan/operator.cpp | 4 ++-- src/query/v2/plan/rewrite/index_lookup.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index e9c8652a2..a5ca9d061 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -508,7 +508,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { // Evaluate the expressions that hold the PrimaryKey. ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.request_router, - storage::v3::View::OLD); + storage::v3::View::NEW); std::vector pk; MG_ASSERT(primary_key_); @@ -539,7 +539,7 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { } if (current_vertex_it_ == current_batch_.end() && - (request_state_ == State::COMPLETED || !MakeRequest(request_router, context))) { + (request_state_ == State::COMPLETED || !MakeRequestSingleFrame(frame, request_router, context))) { ResetExecutionState(); continue; } diff --git a/src/query/v2/plan/rewrite/index_lookup.hpp b/src/query/v2/plan/rewrite/index_lookup.hpp index e134dd49b..f2a0f1bc6 100644 --- a/src/query/v2/plan/rewrite/index_lookup.hpp +++ b/src/query/v2/plan/rewrite/index_lookup.hpp @@ -600,7 +600,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { EraseLabelFilters(node_symbol, prim_label); std::vector pk_expressions; std::transform(primary_key.begin(), primary_key.end(), std::back_inserter(pk_expressions), - [](const auto &exp) { return exp.first; }); + [](const auto &exp) { return exp.second.property_filter->value_; }); return std::make_unique(input, node_symbol, GetLabel(prim_label), pk_expressions); } } From ace1eb401f4aab75a5b955849957dcb96f149897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Sun, 15 Jan 2023 18:25:32 +0100 Subject: [PATCH 041/251] Make unit tests compile with new gtest version --- tests/unit/query_v2_create_node_multiframe.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp index f19082783..39ce17b43 100644 --- a/tests/unit/query_v2_create_node_multiframe.cpp +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -55,11 +55,7 @@ TEST(CreateNodeTest, CreateNodeCursor) { p.push_back(std::make_pair(msgs::PropertyId::FromUint(2), &literal)); node.properties.emplace<0>(std::move(p)); - auto once_cur = plan::MakeUniqueCursorPtr(utils::NewDeleteResource()); - EXPECT_CALL(BaseToMock(once_cur.get()), PullMultiple(_, _)).Times(1); - - std::shared_ptr once_op = std::make_shared(); - EXPECT_CALL(BaseToMock(once_op.get()), MakeCursor(_)).Times(1).WillOnce(Return(std::move(once_cur))); + auto once_op = std::make_shared(); auto create_expand = plan::CreateNode(once_op, node); auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); From b2b9b1d5cbc06a110f34bbb250591ecb5d5d22e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Sun, 15 Jan 2023 18:25:48 +0100 Subject: [PATCH 042/251] Eliminate warnings about deprecated methods --- tests/unit/bfs_single_node.cpp | 32 +++++++++---------- tests/unit/cypher_main_visitor.cpp | 6 ++-- ..._print_ast_to_original_expression_test.cpp | 4 +-- tests/unit/query_plan.cpp | 4 +-- .../query_v2_create_expand_multiframe.cpp | 3 +- tests/unit/query_v2_cypher_main_visitor.cpp | 6 ++-- tests/unit/storage_v3.cpp | 6 ++-- tests/unit/storage_v3_edge.cpp | 6 ++-- tests/unit/storage_v3_isolation_level.cpp | 6 ++-- tests/unit/utils_csv_parsing.cpp | 4 +-- tests/unit/utils_file_locker.cpp | 8 ++--- tests/unit/utils_memory.cpp | 4 +-- 12 files changed, 44 insertions(+), 45 deletions(-) diff --git a/tests/unit/bfs_single_node.cpp b/tests/unit/bfs_single_node.cpp index 93002eef5..eab4f3973 100644 --- a/tests/unit/bfs_single_node.cpp +++ b/tests/unit/bfs_single_node.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -88,14 +88,14 @@ TEST_P(SingleNodeBfsTest, All) { std::unique_ptr SingleNodeBfsTest::db_{nullptr}; -INSTANTIATE_TEST_CASE_P(DirectionAndExpansionDepth, SingleNodeBfsTest, - testing::Combine(testing::Range(-1, kVertexCount), testing::Range(-1, kVertexCount), - testing::Values(EdgeAtom::Direction::OUT, EdgeAtom::Direction::IN, - EdgeAtom::Direction::BOTH), - testing::Values(std::vector{}), testing::Bool(), - testing::Values(FilterLambdaType::NONE))); +INSTANTIATE_TEST_SUITE_P(DirectionAndExpansionDepth, SingleNodeBfsTest, + testing::Combine(testing::Range(-1, kVertexCount), testing::Range(-1, kVertexCount), + testing::Values(EdgeAtom::Direction::OUT, EdgeAtom::Direction::IN, + EdgeAtom::Direction::BOTH), + testing::Values(std::vector{}), testing::Bool(), + testing::Values(FilterLambdaType::NONE))); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( EdgeType, SingleNodeBfsTest, testing::Combine(testing::Values(-1), testing::Values(-1), testing::Values(EdgeAtom::Direction::OUT, EdgeAtom::Direction::IN, EdgeAtom::Direction::BOTH), @@ -103,11 +103,11 @@ INSTANTIATE_TEST_CASE_P( std::vector{"b"}, std::vector{"a", "b"}), testing::Bool(), testing::Values(FilterLambdaType::NONE))); -INSTANTIATE_TEST_CASE_P(FilterLambda, SingleNodeBfsTest, - testing::Combine(testing::Values(-1), testing::Values(-1), - testing::Values(EdgeAtom::Direction::OUT, EdgeAtom::Direction::IN, - EdgeAtom::Direction::BOTH), - testing::Values(std::vector{}), testing::Bool(), - testing::Values(FilterLambdaType::NONE, FilterLambdaType::USE_FRAME, - FilterLambdaType::USE_FRAME_NULL, FilterLambdaType::USE_CTX, - FilterLambdaType::ERROR))); +INSTANTIATE_TEST_SUITE_P(FilterLambda, SingleNodeBfsTest, + testing::Combine(testing::Values(-1), testing::Values(-1), + testing::Values(EdgeAtom::Direction::OUT, EdgeAtom::Direction::IN, + EdgeAtom::Direction::BOTH), + testing::Values(std::vector{}), testing::Bool(), + testing::Values(FilterLambdaType::NONE, FilterLambdaType::USE_FRAME, + FilterLambdaType::USE_FRAME_NULL, FilterLambdaType::USE_CTX, + FilterLambdaType::ERROR))); diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index 40aeb0161..93f0653b4 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -294,7 +294,7 @@ std::shared_ptr gAstGeneratorTypes[] = { std::make_shared(), }; -INSTANTIATE_TEST_CASE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::ValuesIn(gAstGeneratorTypes)); +INSTANTIATE_TEST_SUITE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::ValuesIn(gAstGeneratorTypes)); // NOTE: The above used to use *Typed Tests* functionality of gtest library. // Unfortunately, the compilation time of this test increased to full 2 minutes! @@ -308,7 +308,7 @@ INSTANTIATE_TEST_CASE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::Val // ClonedAstGenerator, CachedAstGenerator> // AstGeneratorTypes; // -// TYPED_TEST_CASE(CypherMainVisitorTest, AstGeneratorTypes); +// TYPED_TEST_SUITE(CypherMainVisitorTest, AstGeneratorTypes); TEST_P(CypherMainVisitorTest, SyntaxException) { auto &ast_generator = *GetParam(); diff --git a/tests/unit/pretty_print_ast_to_original_expression_test.cpp b/tests/unit/pretty_print_ast_to_original_expression_test.cpp index e5d77ae0e..a2144c8aa 100644 --- a/tests/unit/pretty_print_ast_to_original_expression_test.cpp +++ b/tests/unit/pretty_print_ast_to_original_expression_test.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -67,7 +67,7 @@ TEST_P(ExpressiontoStringTest, Example) { EXPECT_EQ(rewritten_expression, rewritten_expression2); } -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( PARAMETER, ExpressiontoStringTest, ::testing::Values( std::make_pair(std::string("2 / 1"), std::string("(2 / 1)")), diff --git a/tests/unit/query_plan.cpp b/tests/unit/query_plan.cpp index 93d2f33c7..935709cec 100644 --- a/tests/unit/query_plan.cpp +++ b/tests/unit/query_plan.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -90,7 +90,7 @@ void DeleteListContent(std::list *list) { delete ptr; } } -TYPED_TEST_CASE(TestPlanner, PlannerTypes); +TYPED_TEST_SUITE(TestPlanner, PlannerTypes); TYPED_TEST(TestPlanner, MatchNodeReturn) { // Test MATCH (n) RETURN n diff --git a/tests/unit/query_v2_create_expand_multiframe.cpp b/tests/unit/query_v2_create_expand_multiframe.cpp index 8720656fd..e978a6bb5 100644 --- a/tests/unit/query_v2_create_expand_multiframe.cpp +++ b/tests/unit/query_v2_create_expand_multiframe.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -63,7 +63,6 @@ TEST(CreateExpandTest, Cursor) { node.symbol = symbol_table.CreateSymbol("u", true); auto once_op = std::make_shared(); - auto once_cur = once_op->MakeCursor(utils::NewDeleteResource()); auto create_expand = plan::CreateExpand(node, edge, once_op, src, true); auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); diff --git a/tests/unit/query_v2_cypher_main_visitor.cpp b/tests/unit/query_v2_cypher_main_visitor.cpp index d7e4169e2..4d5311a6e 100644 --- a/tests/unit/query_v2_cypher_main_visitor.cpp +++ b/tests/unit/query_v2_cypher_main_visitor.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -299,7 +299,7 @@ std::shared_ptr gAstGeneratorTypes[] = { std::make_shared(), }; -INSTANTIATE_TEST_CASE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::ValuesIn(gAstGeneratorTypes)); +INSTANTIATE_TEST_SUITE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::ValuesIn(gAstGeneratorTypes)); // NOTE: The above used to use *Typed Tests* functionality of gtest library. // Unfortunately, the compilation time of this test increased to full 2 minutes! @@ -313,7 +313,7 @@ INSTANTIATE_TEST_CASE_P(AstGeneratorTypes, CypherMainVisitorTest, ::testing::Val // ClonedAstGenerator, CachedAstGenerator> // AstGeneratorTypes; // -// TYPED_TEST_CASE(CypherMainVisitorTest, AstGeneratorTypes); +// TYPED_TEST_SUITE(CypherMainVisitorTest, AstGeneratorTypes); TEST_P(CypherMainVisitorTest, SyntaxException) { auto &ast_generator = *GetParam(); diff --git a/tests/unit/storage_v3.cpp b/tests/unit/storage_v3.cpp index 6f6902d1c..1832165f9 100644 --- a/tests/unit/storage_v3.cpp +++ b/tests/unit/storage_v3.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -82,8 +82,8 @@ class StorageV3 : public ::testing::TestWithParam { Config{.gc = {.reclamation_interval = reclamation_interval}}}; coordinator::Hlc last_hlc{0, io::Time{}}; }; -INSTANTIATE_TEST_CASE_P(WithGc, StorageV3, ::testing::Values(true)); -INSTANTIATE_TEST_CASE_P(WithoutGc, StorageV3, ::testing::Values(false)); +INSTANTIATE_TEST_SUITE_P(WithGc, StorageV3, ::testing::Values(true)); +INSTANTIATE_TEST_SUITE_P(WithoutGc, StorageV3, ::testing::Values(false)); // NOLINTNEXTLINE(hicpp-special-member-functions) TEST_P(StorageV3, Commit) { diff --git a/tests/unit/storage_v3_edge.cpp b/tests/unit/storage_v3_edge.cpp index 3d2ab8bbd..8637db11c 100644 --- a/tests/unit/storage_v3_edge.cpp +++ b/tests/unit/storage_v3_edge.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -60,8 +60,8 @@ class StorageEdgeTest : public ::testing::TestWithParam { coordinator::Hlc last_hlc{0, io::Time{}}; }; -INSTANTIATE_TEST_CASE_P(EdgesWithProperties, StorageEdgeTest, ::testing::Values(true)); -INSTANTIATE_TEST_CASE_P(EdgesWithoutProperties, StorageEdgeTest, ::testing::Values(false)); +INSTANTIATE_TEST_SUITE_P(EdgesWithProperties, StorageEdgeTest, ::testing::Values(true)); +INSTANTIATE_TEST_SUITE_P(EdgesWithoutProperties, StorageEdgeTest, ::testing::Values(false)); // NOLINTNEXTLINE(hicpp-special-member-functions) TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { diff --git a/tests/unit/storage_v3_isolation_level.cpp b/tests/unit/storage_v3_isolation_level.cpp index 6b661f632..bd0f27b6b 100644 --- a/tests/unit/storage_v3_isolation_level.cpp +++ b/tests/unit/storage_v3_isolation_level.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -135,6 +135,6 @@ TEST_P(StorageIsolationLevelTest, Visibility) { } } -INSTANTIATE_TEST_CASE_P(ParameterizedStorageIsolationLevelTests, StorageIsolationLevelTest, - ::testing::ValuesIn(isolation_levels), StorageIsolationLevelTest::PrintToStringParamName()); +INSTANTIATE_TEST_SUITE_P(ParameterizedStorageIsolationLevelTests, StorageIsolationLevelTest, + ::testing::ValuesIn(isolation_levels), StorageIsolationLevelTest::PrintToStringParamName()); } // namespace memgraph::storage::v3::tests diff --git a/tests/unit/utils_csv_parsing.cpp b/tests/unit/utils_csv_parsing.cpp index 3c852b171..e8d6c2241 100644 --- a/tests/unit/utils_csv_parsing.cpp +++ b/tests/unit/utils_csv_parsing.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -330,4 +330,4 @@ TEST_P(CsvReaderTest, EmptyColumns) { } } -INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n")); +INSTANTIATE_TEST_SUITE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n")); diff --git a/tests/unit/utils_file_locker.cpp b/tests/unit/utils_file_locker.cpp index 21c71b1a3..f2217953e 100644 --- a/tests/unit/utils_file_locker.cpp +++ b/tests/unit/utils_file_locker.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -190,9 +190,9 @@ TEST_P(FileLockerParameterizedTest, RemovePath) { std::filesystem::current_path(save_path); } -INSTANTIATE_TEST_CASE_P(FileLockerPathVariantTests, FileLockerParameterizedTest, - ::testing::Values(std::make_tuple(false, false), std::make_tuple(false, true), - std::make_tuple(true, false), std::make_tuple(true, true))); +INSTANTIATE_TEST_SUITE_P(FileLockerPathVariantTests, FileLockerParameterizedTest, + ::testing::Values(std::make_tuple(false, false), std::make_tuple(false, true), + std::make_tuple(true, false), std::make_tuple(true, true))); TEST_F(FileLockerTest, MultipleLockers) { CreateFiles(3); diff --git a/tests/unit/utils_memory.cpp b/tests/unit/utils_memory.cpp index 70bf85653..73f78d545 100644 --- a/tests/unit/utils_memory.cpp +++ b/tests/unit/utils_memory.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -393,7 +393,7 @@ class AllocatorTest : public ::testing::Test {}; using ContainersWithAllocators = ::testing::Types; -TYPED_TEST_CASE(AllocatorTest, ContainersWithAllocators); +TYPED_TEST_SUITE(AllocatorTest, ContainersWithAllocators); TYPED_TEST(AllocatorTest, PropagatesToStdUsesAllocator) { std::vector> vec(memgraph::utils::NewDeleteResource()); From b30137ab7ac7bdd5005b6f5ffc23d45131e6fa6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Sun, 15 Jan 2023 18:39:58 +0100 Subject: [PATCH 043/251] Improve unit tests to catch bug --- src/query/v2/requests.hpp | 3 ++- tests/unit/query_v2_create_node_multiframe.cpp | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/query/v2/requests.hpp b/src/query/v2/requests.hpp index 9ff9a1bae..2335fea7d 100644 --- a/src/query/v2/requests.hpp +++ b/src/query/v2/requests.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -36,6 +36,7 @@ struct Value; struct Label { LabelId id; friend bool operator==(const Label &lhs, const Label &rhs) { return lhs.id == rhs.id; } + friend bool operator==(const Label &lhs, const LabelId &rhs) { return lhs.id == rhs; } }; // TODO(kostasrim) update this with CompoundKey, same for the rest of the file. diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp index 39ce17b43..5e5d83249 100644 --- a/tests/unit/query_v2_create_node_multiframe.cpp +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -36,15 +36,13 @@ MultiFrame CreateMultiFrame(const size_t max_pos) { TEST(CreateNodeTest, CreateNodeCursor) { using testing::_; + using testing::ElementsAre; using testing::Return; AstStorage ast; SymbolTable symbol_table; plan::NodeCreationInfo node; - plan::EdgeCreationInfo edge; - edge.edge_type = msgs::EdgeTypeId::FromUint(1); - edge.direction = EdgeAtom::Direction::IN; auto id_alloc = IdAllocator(0, 100); node.symbol = symbol_table.CreateSymbol("n", true); @@ -67,5 +65,19 @@ TEST(CreateNodeTest, CreateNodeCursor) { auto context = MakeContext(ast, symbol_table, &router, &id_alloc); auto multi_frame = CreateMultiFrame(context.symbol_table.max_position()); cursor->PullMultiple(multi_frame, context); + + auto frames = multi_frame.GetValidFramesReader(); + auto number_of_valid_frames = 0; + for (auto &frame : frames) { + ++number_of_valid_frames; + EXPECT_EQ(frame[node.symbol].IsEdge(), true); + const auto &n = frame[node.symbol].ValueVertex(); + EXPECT_THAT(n.Labels(), ElementsAre(msgs::Label{msgs::LabelId::FromUint(2)})); + } + EXPECT_EQ(number_of_valid_frames, 1); + + auto invalid_frames = multi_frame.GetInvalidFramesPopulator(); + auto number_of_invalid_frames = std::distance(invalid_frames.begin(), invalid_frames.end()); + EXPECT_EQ(number_of_invalid_frames, 99); } } // namespace memgraph::query::v2 From c139856b2a9b17ea52bc9c38107eeaa7f3b966f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Sun, 15 Jan 2023 18:52:36 +0100 Subject: [PATCH 044/251] Fix unit tests --- src/query/v2/plan/operator.cpp | 14 +++++++------- tests/unit/query_v2_create_node_multiframe.cpp | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index da282387c..630599276 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -192,7 +192,7 @@ class DistributedCreateNodeCursor : public Cursor { void PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) override { SCOPED_PROFILE_OP("CreateNodeMF"); input_cursor_->PullMultiple(multi_frame, context); - auto &request_router = context.request_router; + auto *request_router = context.request_router; { SCOPED_REQUEST_WAIT_PROFILE; request_router->CreateVertices(NodeCreationInfoToRequests(context, multi_frame)); @@ -259,10 +259,10 @@ class DistributedCreateNodeCursor : public Cursor { } void PlaceNodesOnTheMultiFrame(MultiFrame &multi_frame, ExecutionContext &context) { - auto multi_frame_reader = multi_frame.GetValidFramesConsumer(); + auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); size_t i = 0; - MG_ASSERT(std::distance(multi_frame_reader.begin(), multi_frame_reader.end())); - for (auto &frame : multi_frame_reader) { + MG_ASSERT(std::distance(multi_frame_modifier.begin(), multi_frame_modifier.end())); + for (auto &frame : multi_frame_modifier) { const auto primary_label = msgs::Label{.id = nodes_info_[0]->labels[0]}; msgs::Vertex v{.id = std::make_pair(primary_label, primary_keys_[i])}; frame[nodes_info_.front()->symbol] = TypedValue( @@ -272,8 +272,8 @@ class DistributedCreateNodeCursor : public Cursor { std::vector NodeCreationInfoToRequests(ExecutionContext &context, MultiFrame &multi_frame) { std::vector requests; - auto multi_frame_reader = multi_frame.GetValidFramesConsumer(); - for (auto &frame : multi_frame_reader) { + auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); + for (auto &frame : multi_frame_modifier) { msgs::PrimaryKey pk; for (const auto &node_info : nodes_info_) { msgs::NewVertex rqst; diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp index 5e5d83249..b04b00e96 100644 --- a/tests/unit/query_v2_create_node_multiframe.cpp +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include "gmock/gmock.h" #include "mock_helpers.hpp" #include "query/v2/bindings/frame.hpp" @@ -26,17 +27,13 @@ namespace memgraph::query::v2 { MultiFrame CreateMultiFrame(const size_t max_pos) { static constexpr size_t frame_size = 100; MultiFrame multi_frame(max_pos, frame_size, utils::NewDeleteResource()); - auto frames_populator = multi_frame.GetInvalidFramesPopulator(); - for (auto &frame : frames_populator) { - frame.MakeValid(); - } return multi_frame; } TEST(CreateNodeTest, CreateNodeCursor) { using testing::_; - using testing::ElementsAre; + using testing::IsEmpty; using testing::Return; AstStorage ast; @@ -46,11 +43,12 @@ TEST(CreateNodeTest, CreateNodeCursor) { auto id_alloc = IdAllocator(0, 100); node.symbol = symbol_table.CreateSymbol("n", true); - node.labels.push_back(msgs::LabelId::FromUint(2)); + const auto primary_label_id = msgs::LabelId::FromUint(2); + node.labels.push_back(primary_label_id); auto literal = PrimitiveLiteral(); literal.value_ = TypedValue(static_cast(200)); auto p = plan::PropertiesMapList{}; - p.push_back(std::make_pair(msgs::PropertyId::FromUint(2), &literal)); + p.push_back(std::make_pair(msgs::PropertyId::FromUint(3), &literal)); node.properties.emplace<0>(std::move(p)); auto once_op = std::make_shared(); @@ -70,9 +68,11 @@ TEST(CreateNodeTest, CreateNodeCursor) { auto number_of_valid_frames = 0; for (auto &frame : frames) { ++number_of_valid_frames; - EXPECT_EQ(frame[node.symbol].IsEdge(), true); + EXPECT_EQ(frame[node.symbol].IsVertex(), true); const auto &n = frame[node.symbol].ValueVertex(); - EXPECT_THAT(n.Labels(), ElementsAre(msgs::Label{msgs::LabelId::FromUint(2)})); + EXPECT_THAT(n.Labels(), IsEmpty()); + EXPECT_EQ(n.PrimaryLabel(), primary_label_id); + // TODO(antaljanosbenjamin): Check primary key } EXPECT_EQ(number_of_valid_frames, 1); From e40f7f507b422ddb0daa4379bd529fce2234cfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 16 Jan 2023 08:40:43 +0100 Subject: [PATCH 045/251] Fix pull logic for multiframe --- src/query/v2/interpreter.cpp | 4 ++-- src/query/v2/multiframe.cpp | 6 +++++- src/query/v2/multiframe.hpp | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index fde11ac00..594942aec 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -734,7 +734,7 @@ std::optional PullPlan::PullMultiple(AnyStrea // Returns true if a result was pulled. const auto pull_result = [&]() -> bool { cursor_->PullMultiple(multi_frame_, ctx_); - return multi_frame_.HasValidFrame(); + return !multi_frame_.HasInvalidFrame(); }; const auto stream_values = [&output_symbols, &stream](const Frame &frame) { diff --git a/src/query/v2/multiframe.cpp b/src/query/v2/multiframe.cpp index 2cb591153..0ddfd3aa7 100644 --- a/src/query/v2/multiframe.cpp +++ b/src/query/v2/multiframe.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -48,6 +48,10 @@ bool MultiFrame::HasValidFrame() const noexcept { return std::any_of(frames_.begin(), frames_.end(), [](auto &frame) { return frame.IsValid(); }); } +bool MultiFrame::HasInvalidFrame() const noexcept { + return std::any_of(frames_.rbegin(), frames_.rend(), [](auto &frame) { return !frame.IsValid(); }); +} + // NOLINTNEXTLINE (bugprone-exception-escape) void MultiFrame::DefragmentValidFrames() noexcept { /* diff --git a/src/query/v2/multiframe.hpp b/src/query/v2/multiframe.hpp index 7d9b73700..b46be171a 100644 --- a/src/query/v2/multiframe.hpp +++ b/src/query/v2/multiframe.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -81,6 +81,7 @@ class MultiFrame { void MakeAllFramesInvalid() noexcept; bool HasValidFrame() const noexcept; + bool HasInvalidFrame() const noexcept; inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); } From 392f6e2b730f644f7707a7861b81dcc43ee71280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 16 Jan 2023 08:57:23 +0100 Subject: [PATCH 046/251] Reduce the number of node infos to a maximum of one --- src/query/v2/plan/operator.cpp | 141 +++++++++++++++------------------ 1 file changed, 65 insertions(+), 76 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 630599276..2f191c113 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -170,9 +170,8 @@ uint64_t ComputeProfilingKey(const T *obj) { class DistributedCreateNodeCursor : public Cursor { public: using InputOperator = std::shared_ptr; - DistributedCreateNodeCursor(const InputOperator &op, utils::MemoryResource *mem, - std::vector nodes_info) - : input_cursor_(op->MakeCursor(mem)), nodes_info_(std::move(nodes_info)) {} + DistributedCreateNodeCursor(const InputOperator &op, utils::MemoryResource *mem, const NodeCreationInfo &node_info) + : input_cursor_(op->MakeCursor(mem)), node_info_(node_info) {} bool Pull(Frame &frame, ExecutionContext &context) override { SCOPED_PROFILE_OP("CreateNode"); @@ -206,27 +205,78 @@ class DistributedCreateNodeCursor : public Cursor { void PlaceNodeOnTheFrame(Frame &frame, ExecutionContext &context) { // TODO(kostasrim) Make this work with batching - const auto primary_label = msgs::Label{.id = nodes_info_[0]->labels[0]}; + const auto primary_label = msgs::Label{.id = node_info_.labels[0]}; msgs::Vertex v{.id = std::make_pair(primary_label, primary_keys_[0])}; - frame[nodes_info_.front()->symbol] = + frame[node_info_.symbol] = TypedValue(query::v2::accessors::VertexAccessor(std::move(v), src_vertex_props_[0], context.request_router)); } std::vector NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) { std::vector requests; - // TODO(kostasrim) this assertion should be removed once we support multiple vertex creation - MG_ASSERT(nodes_info_.size() == 1); msgs::PrimaryKey pk; - for (const auto &node_info : nodes_info_) { + msgs::NewVertex rqst; + MG_ASSERT(!node_info_.labels.empty(), "Cannot determine primary label"); + const auto primary_label = node_info_.labels[0]; + // TODO(jbajic) Fix properties not send, + // suggestion: ignore distinction between properties and primary keys + // since schema validation is done on storage side + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, + storage::v3::View::NEW); + if (const auto *node_info_properties = std::get_if(&node_info_.properties)) { + for (const auto &[key, value_expression] : *node_info_properties) { + TypedValue val = value_expression->Accept(evaluator); + if (context.request_router->IsPrimaryKey(primary_label, key)) { + rqst.primary_key.push_back(TypedValueToValue(val)); + pk.push_back(TypedValueToValue(val)); + } + } + } else { + auto property_map = evaluator.Visit(*std::get(node_info_.properties)).ValueMap(); + for (const auto &[key, value] : property_map) { + auto key_str = std::string(key); + auto property_id = context.request_router->NameToProperty(key_str); + if (context.request_router->IsPrimaryKey(primary_label, property_id)) { + rqst.primary_key.push_back(TypedValueToValue(value)); + pk.push_back(TypedValueToValue(value)); + } + } + } + + // TODO(kostasrim) Copy non primary labels as well + rqst.label_ids.push_back(msgs::Label{.id = primary_label}); + src_vertex_props_.push_back(rqst.properties); + requests.push_back(std::move(rqst)); + + primary_keys_.push_back(std::move(pk)); + return requests; + } + + void PlaceNodesOnTheMultiFrame(MultiFrame &multi_frame, ExecutionContext &context) { + auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); + size_t i = 0; + MG_ASSERT(std::distance(multi_frame_modifier.begin(), multi_frame_modifier.end())); + for (auto &frame : multi_frame_modifier) { + const auto primary_label = msgs::Label{.id = node_info_.labels[0]}; + msgs::Vertex v{.id = std::make_pair(primary_label, primary_keys_[i])}; + frame[node_info_.symbol] = TypedValue( + query::v2::accessors::VertexAccessor(std::move(v), src_vertex_props_[i++], context.request_router)); + } + } + + std::vector NodeCreationInfoToRequests(ExecutionContext &context, MultiFrame &multi_frame) { + std::vector requests; + auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); + for (auto &frame : multi_frame_modifier) { + msgs::PrimaryKey pk; msgs::NewVertex rqst; - MG_ASSERT(!node_info->labels.empty(), "Cannot determine primary label"); - const auto primary_label = node_info->labels[0]; + MG_ASSERT(!node_info_.labels.empty(), "Cannot determine primary label"); + const auto primary_label = node_info_.labels[0]; // TODO(jbajic) Fix properties not send, // suggestion: ignore distinction between properties and primary keys // since schema validation is done on storage side ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, storage::v3::View::NEW); - if (const auto *node_info_properties = std::get_if(&node_info->properties)) { + if (const auto *node_info_properties = std::get_if(&node_info_.properties)) { for (const auto &[key, value_expression] : *node_info_properties) { TypedValue val = value_expression->Accept(evaluator); if (context.request_router->IsPrimaryKey(primary_label, key)) { @@ -235,7 +285,7 @@ class DistributedCreateNodeCursor : public Cursor { } } } else { - auto property_map = evaluator.Visit(*std::get(node_info->properties)).ValueMap(); + auto property_map = evaluator.Visit(*std::get(node_info_.properties)).ValueMap(); for (const auto &[key, value] : property_map) { auto key_str = std::string(key); auto property_id = context.request_router->NameToProperty(key_str); @@ -246,80 +296,19 @@ class DistributedCreateNodeCursor : public Cursor { } } - if (node_info->labels.empty()) { - throw QueryRuntimeException("Primary label must be defined!"); - } // TODO(kostasrim) Copy non primary labels as well rqst.label_ids.push_back(msgs::Label{.id = primary_label}); src_vertex_props_.push_back(rqst.properties); requests.push_back(std::move(rqst)); - } - primary_keys_.push_back(std::move(pk)); - return requests; - } - - void PlaceNodesOnTheMultiFrame(MultiFrame &multi_frame, ExecutionContext &context) { - auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); - size_t i = 0; - MG_ASSERT(std::distance(multi_frame_modifier.begin(), multi_frame_modifier.end())); - for (auto &frame : multi_frame_modifier) { - const auto primary_label = msgs::Label{.id = nodes_info_[0]->labels[0]}; - msgs::Vertex v{.id = std::make_pair(primary_label, primary_keys_[i])}; - frame[nodes_info_.front()->symbol] = TypedValue( - query::v2::accessors::VertexAccessor(std::move(v), src_vertex_props_[i++], context.request_router)); - } - } - - std::vector NodeCreationInfoToRequests(ExecutionContext &context, MultiFrame &multi_frame) { - std::vector requests; - auto multi_frame_modifier = multi_frame.GetValidFramesModifier(); - for (auto &frame : multi_frame_modifier) { - msgs::PrimaryKey pk; - for (const auto &node_info : nodes_info_) { - msgs::NewVertex rqst; - MG_ASSERT(!node_info->labels.empty(), "Cannot determine primary label"); - const auto primary_label = node_info->labels[0]; - // TODO(jbajic) Fix properties not send, - // suggestion: ignore distinction between properties and primary keys - // since schema validation is done on storage side - ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr, - storage::v3::View::NEW); - if (const auto *node_info_properties = std::get_if(&node_info->properties)) { - for (const auto &[key, value_expression] : *node_info_properties) { - TypedValue val = value_expression->Accept(evaluator); - if (context.request_router->IsPrimaryKey(primary_label, key)) { - rqst.primary_key.push_back(TypedValueToValue(val)); - pk.push_back(TypedValueToValue(val)); - } - } - } else { - auto property_map = evaluator.Visit(*std::get(node_info->properties)).ValueMap(); - for (const auto &[key, value] : property_map) { - auto key_str = std::string(key); - auto property_id = context.request_router->NameToProperty(key_str); - if (context.request_router->IsPrimaryKey(primary_label, property_id)) { - rqst.primary_key.push_back(TypedValueToValue(value)); - pk.push_back(TypedValueToValue(value)); - } - } - } - - if (node_info->labels.empty()) { - throw QueryRuntimeException("Primary label must be defined!"); - } - // TODO(kostasrim) Copy non primary labels as well - rqst.label_ids.push_back(msgs::Label{.id = primary_label}); - src_vertex_props_.push_back(rqst.properties); - requests.push_back(std::move(rqst)); - } primary_keys_.push_back(std::move(pk)); } + return requests; } private: const UniqueCursorPtr input_cursor_; - std::vector nodes_info_; + NodeCreationInfo node_info_; std::vector>> src_vertex_props_; std::vector primary_keys_; }; @@ -364,7 +353,7 @@ ACCEPT_WITH_INPUT(CreateNode) UniqueCursorPtr CreateNode::MakeCursor(utils::MemoryResource *mem) const { EventCounter::IncrementCounter(EventCounter::CreateNodeOperator); - return MakeUniqueCursorPtr(mem, input_, mem, std::vector{&this->node_info_}); + return MakeUniqueCursorPtr(mem, input_, mem, this->node_info_); } std::vector CreateNode::ModifiedSymbols(const SymbolTable &table) const { From 920ad277a5e44a34083f0030d3618695a7eb42bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 16 Jan 2023 09:03:35 +0100 Subject: [PATCH 047/251] Add assertion about primary label --- src/query/v2/plan/operator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 2f191c113..4ea189772 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -271,6 +271,7 @@ class DistributedCreateNodeCursor : public Cursor { msgs::NewVertex rqst; MG_ASSERT(!node_info_.labels.empty(), "Cannot determine primary label"); const auto primary_label = node_info_.labels[0]; + MG_ASSERT(context.request_router->IsPrimaryLabel(primary_label), "First label has to be a primary label!"); // TODO(jbajic) Fix properties not send, // suggestion: ignore distinction between properties and primary keys // since schema validation is done on storage side From 6de683d7f9b6367a2fcb6c24a3cf0db6e0a0bde3 Mon Sep 17 00:00:00 2001 From: jbajic Date: Mon, 16 Jan 2023 09:51:06 +0100 Subject: [PATCH 048/251] Ignore commited transactions --- src/storage/v3/shard.cpp | 5 ++++- tests/unit/storage_v3_shard_split.cpp | 29 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index 803678c8a..d4191b4bb 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -1179,7 +1179,10 @@ std::map Shard::CollectTransactions(const std::set transactions; for (const auto commit_start : collected_transactions_start_id) { - transactions.insert({commit_start, start_logical_id_to_transaction_.at(commit_start)->Clone()}); + // If it does not contain then the transaction has commited, and we ignore it + if (start_logical_id_to_transaction_.contains(commit_start)) { + transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()}); + } } // It is necessary to clone all the transactions first so we have new addresses // for deltas, before doing alignment of deltas and prev_ptr diff --git a/tests/unit/storage_v3_shard_split.cpp b/tests/unit/storage_v3_shard_split.cpp index ebfca2921..1cca21d62 100644 --- a/tests/unit/storage_v3_shard_split.cpp +++ b/tests/unit/storage_v3_shard_split.cpp @@ -87,7 +87,7 @@ TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) { EXPECT_EQ(splitted_data.transactions.size(), 0); } -TEST_F(ShardSplitTest, TestBasicSplit) { +TEST_F(ShardSplitTest, TestBasicSplitBeforeCommit) { auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError()); EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError()); @@ -112,4 +112,31 @@ TEST_F(ShardSplitTest, TestBasicSplit) { EXPECT_EQ(splitted_data.transactions.size(), 1); } +TEST_F(ShardSplitTest, TestBasicSplitAfterCommit) { + auto acc = storage.Access(GetNextHlc()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError()); + EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError()); + + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1)) + .HasError()); + EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}}, + VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2)) + .HasError()); + + acc.Commit(GetNextHlc()); + + auto splitted_data = storage.PerformSplit({PropertyValue(4)}); + EXPECT_EQ(splitted_data.vertices.size(), 3); + EXPECT_EQ(splitted_data.edges->size(), 2); + EXPECT_EQ(splitted_data.transactions.size(), 0); +} + } // namespace memgraph::storage::v3::tests From 775e950dbae0bc34339c91bd9def081c89f66308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 16 Jan 2023 10:16:12 +0100 Subject: [PATCH 049/251] Update unit tests to test the new logic --- tests/unit/query_v2_create_node_multiframe.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/query_v2_create_node_multiframe.cpp b/tests/unit/query_v2_create_node_multiframe.cpp index b04b00e96..014fff6ce 100644 --- a/tests/unit/query_v2_create_node_multiframe.cpp +++ b/tests/unit/query_v2_create_node_multiframe.cpp @@ -56,10 +56,9 @@ TEST(CreateNodeTest, CreateNodeCursor) { auto create_expand = plan::CreateNode(once_op, node); auto cursor = create_expand.MakeCursor(utils::NewDeleteResource()); MockedRequestRouter router; - EXPECT_CALL(router, CreateVertices(testing::_)) - .Times(1) - .WillOnce(::testing::Return(std::vector{})); - EXPECT_CALL(router, IsPrimaryKey(testing::_, testing::_)).WillRepeatedly(::testing::Return(true)); + EXPECT_CALL(router, CreateVertices(_)).Times(1).WillOnce(Return(std::vector{})); + EXPECT_CALL(router, IsPrimaryLabel(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(router, IsPrimaryKey(_, _)).WillRepeatedly(Return(true)); auto context = MakeContext(ast, symbol_table, &router, &id_alloc); auto multi_frame = CreateMultiFrame(context.symbol_table.max_position()); cursor->PullMultiple(multi_frame, context); From fdd89e0e81be9d6f22804482b1fa48ae5f4540d2 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 16 Jan 2023 11:27:41 +0100 Subject: [PATCH 050/251] Replace ScanVertices with GetProperties request, in the case of ScanAllByPrimaryKey operator --- src/query/v2/plan/operator.cpp | 36 ++++++++++++++++++++++++++++++++- src/query/v2/request_router.hpp | 3 +-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index a5ca9d061..a7a3efe3b 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -516,7 +516,41 @@ class DistributedScanAllByPrimaryKeyCursor : public Cursor { pk.push_back(TypedValueToValue(primary_key->Accept(evaluator))); } - current_batch_ = request_router.ScanVertices(request_label, pk); + // Original + // current_batch_ = request_router.ScanVertices(request_label, pk); + + // VertexAccessor(Vertex v, std::vector> props, + // const RequestRouterInterface *request_router); + + // VertexAccessor(Vertex v, std::map &&props, const RequestRouterInterface *request_router); + // VertexAccessor(Vertex v, const std::map &props, const RequestRouterInterface + // *request_router); + + // struct Vertex { + // VertexId id; + // std::vector