From 50c485fe4044bf9ee59424e23dcf1a02f843c77f Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 6 Nov 2023 12:37:48 +0100 Subject: [PATCH 01/52] Add storage side capabilites to retrieve metadata In order to get the required metadata in constant time we need to keep track of the node labels and edge types that were ever present in the database. This is done by the two axuiliary datastructures that are present in the storage instances. The ability to get this metadata is propagated to the DBAccessor class, which the query modules can interact with. --- src/query/db_accessor.hpp | 7 +++++++ src/storage/v2/disk/storage.cpp | 1 + src/storage/v2/inmemory/storage.cpp | 3 +++ src/storage/v2/storage.cpp | 18 ++++++++++++++++++ src/storage/v2/storage.hpp | 16 ++++++++++++++++ src/storage/v2/vertex_accessor.cpp | 1 + 6 files changed, 46 insertions(+) diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index d6114edaf..866da0af9 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -597,6 +597,13 @@ class DbAccessor final { return accessor_->ApproximateVertexCount(label, property, lower, upper); } + std::vector ListAllPossiblyPresentVertexLabels() const { + return accessor_->ListAllPossiblyPresentVertexLabels(); + } + std::vector ListAllPossiblyPresentEdgeTypes() const { + return accessor_->ListAllPossiblyPresentEdgeTypes(); + } + storage::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); } storage::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); } diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 809d744ed..f0280fdc0 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -944,6 +944,7 @@ Result DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT); transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN); + storage_->stored_edge_types_.insert(edge_type); storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel); return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_); diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index d2ce80417..22d2f3c16 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -331,6 +331,7 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso if (to_vertex->deleted) return Error::DELETED_OBJECT; } + storage_->stored_edge_types_.insert(edge_type); auto *mem_storage = static_cast(storage_); auto gid = storage::Gid::FromUint(mem_storage->edge_id_.fetch_add(1, std::memory_order_acq_rel)); EdgeRef edge(gid); @@ -395,6 +396,8 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAcces if (to_vertex->deleted) return Error::DELETED_OBJECT; } + storage_->stored_edge_types_.insert(edge_type); + // NOTE: When we update the next `edge_id_` here we perform a RMW // (read-modify-write) operation that ISN'T atomic! But, that isn't an issue // because this function is only called from the replication delta applier diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index ec988285a..5b935c029 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -122,6 +122,24 @@ std::optional Storage::Accessor::GetTransactionId() const { return {}; } +std::vector Storage::Accessor::ListAllPossiblyPresentVertexLabels() const { + std::vector vertex_labels; + vertex_labels.reserve(storage_->stored_node_labels_.size()); + for (const auto label : storage_->stored_node_labels_) { + vertex_labels.emplace_back(LabelToName(label)); + } + return vertex_labels; +} + +std::vector Storage::Accessor::ListAllPossiblyPresentEdgeTypes() const { + std::vector edge_types; + edge_types.reserve(storage_->stored_edge_types_.size()); + for (const auto edge_type : storage_->stored_edge_types_) { + edge_types.emplace_back(EdgeTypeToName(edge_type)); + } + return edge_types; +} + void Storage::Accessor::AdvanceCommand() { transaction_.manyDeltasCache.Clear(); // TODO: Just invalidate the View::OLD cache, NEW should still be fine ++transaction_.command_id; diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 2d36d202a..f3253267e 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -237,6 +237,10 @@ class Storage { const std::string &id() const { return storage_->id(); } + std::vector ListAllPossiblyPresentVertexLabels() const; + + std::vector ListAllPossiblyPresentEdgeTypes() const; + virtual utils::BasicResult CreateIndex(LabelId label) = 0; virtual utils::BasicResult CreateIndex(LabelId label, PropertyId property) = 0; @@ -384,6 +388,18 @@ class Storage { Indices indices_; Constraints constraints_; + // Datastructures to provide fast retrieval of node-label and + // edge-type related metadata. + // Currently we should not remove any node-labels or edge-types even + // if the set of given types are currently not present in the + // database. This metadata is usually used by client side + // applications that want to be aware of the kind of data that *may* + // be present in the database. + + // TODO(gvolfing): check if this would be faster with flat_maps. + std::unordered_set stored_node_labels_; + std::unordered_set stored_edge_types_; + std::atomic vertex_id_{0}; std::atomic edge_id_{0}; const std::string id_; //!< High-level assigned ID diff --git a/src/storage/v2/vertex_accessor.cpp b/src/storage/v2/vertex_accessor.cpp index 924c305ad..91ffd547e 100644 --- a/src/storage/v2/vertex_accessor.cpp +++ b/src/storage/v2/vertex_accessor.cpp @@ -109,6 +109,7 @@ Result VertexAccessor::AddLabel(LabelId label) { CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label); vertex_->labels.push_back(label); + storage_->stored_node_labels_.insert(label); /// TODO: some by pointers, some by reference => not good, make it better storage_->constraints_.unique_constraints_->UpdateOnAddLabel(label, *vertex_, transaction_->start_timestamp); From c4d9116c9ccac4715bd42ec057c9f2030a302819 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 7 Nov 2023 09:35:28 +0100 Subject: [PATCH 02/52] Add queries to obtain the labels and edge types Add two queries to be able to retrieve the labels and edge types this is done through additions to the DatabaseInfoQuery query types. --- src/query/frontend/ast/ast.hpp | 2 +- .../frontend/ast/cypher_main_visitor.cpp | 8 +++++ .../frontend/opencypher/grammar/Cypher.g4 | 6 +++- .../opencypher/grammar/MemgraphCypher.g4 | 1 + .../opencypher/grammar/MemgraphCypherLexer.g4 | 1 + .../frontend/semantic/required_privileges.cpp | 3 ++ src/query/interpreter.cpp | 32 +++++++++++++++++++ 7 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index 6b7fdb2c6..0b7061a4e 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -2932,7 +2932,7 @@ class DatabaseInfoQuery : public memgraph::query::Query { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - enum class InfoType { INDEX, CONSTRAINT }; + enum class InfoType { INDEX, CONSTRAINT, EDGE_TYPES, NODE_LABELS }; DEFVISITABLE(QueryVisitor); diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 6b75061fb..a1a94a9c3 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -124,6 +124,14 @@ antlrcpp::Any CypherMainVisitor::visitDatabaseInfoQuery(MemgraphCypher::Database info_query->info_type_ = DatabaseInfoQuery::InfoType::CONSTRAINT; return info_query; } + if (ctx->edgetypeInfo()) { + info_query->info_type_ = DatabaseInfoQuery::InfoType::EDGE_TYPES; + return info_query; + } + if (ctx->nodelabelInfo()) { + info_query->info_type_ = DatabaseInfoQuery::InfoType::NODE_LABELS; + return info_query; + } // Should never get here throw utils::NotYetImplemented("Database info query: '{}'", ctx->getText()); } diff --git a/src/query/frontend/opencypher/grammar/Cypher.g4 b/src/query/frontend/opencypher/grammar/Cypher.g4 index 53f1fc765..8dfde7c21 100644 --- a/src/query/frontend/opencypher/grammar/Cypher.g4 +++ b/src/query/frontend/opencypher/grammar/Cypher.g4 @@ -47,9 +47,13 @@ indexInfo : INDEX INFO ; constraintInfo : CONSTRAINT INFO ; +edgetypeInfo : EDGE_TYPES INFO ; + +nodelabelInfo : NODE_LABELS INFO ; + buildInfo : BUILD INFO ; -databaseInfoQuery : SHOW ( indexInfo | constraintInfo ) ; +databaseInfoQuery : SHOW ( indexInfo | constraintInfo | edgetypeInfo | nodelabelInfo ) ; systemInfoQuery : SHOW ( storageInfo | buildInfo ) ; diff --git a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 index bac189a53..d585acbb1 100644 --- a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 +++ b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 @@ -61,6 +61,7 @@ memgraphCypherKeyword : cypherKeyword | GRANT | HEADER | IDENTIFIED + | NODE_LABELS | NULLIF | IMPORT | INACTIVE diff --git a/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 b/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 index db1a1ae76..1b44a6e79 100644 --- a/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 +++ b/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 @@ -89,6 +89,7 @@ MULTI_DATABASE_EDIT : M U L T I UNDERSCORE D A T A B A S E UNDERSCORE E D I MULTI_DATABASE_USE : M U L T I UNDERSCORE D A T A B A S E UNDERSCORE U S E ; NEXT : N E X T ; NO : N O ; +NODE_LABELS : N O D E UNDERSCORE L A B E L S ; NOTHING : N O T H I N G ; ON_DISK_TRANSACTIONAL : O N UNDERSCORE D I S K UNDERSCORE T R A N S A C T I O N A L ; NULLIF : N U L L I F ; diff --git a/src/query/frontend/semantic/required_privileges.cpp b/src/query/frontend/semantic/required_privileges.cpp index b5b75e26e..04772cded 100644 --- a/src/query/frontend/semantic/required_privileges.cpp +++ b/src/query/frontend/semantic/required_privileges.cpp @@ -38,6 +38,9 @@ class PrivilegeExtractor : public QueryVisitor, public HierarchicalTreeVis void Visit(DatabaseInfoQuery &info_query) override { switch (info_query.info_type_) { case DatabaseInfoQuery::InfoType::INDEX: + // TODO: Reconsider priviliges, this 4 should have the same. + case DatabaseInfoQuery::InfoType::EDGE_TYPES: + case DatabaseInfoQuery::InfoType::NODE_LABELS: // TODO: This should be INDEX | STATS, but we don't have support for // *or* with privileges. AddPrivilege(AuthQuery::Privilege::INDEX); diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 074c90176..9313f4547 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -3076,6 +3076,38 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici }; break; } + case DatabaseInfoQuery::InfoType::EDGE_TYPES: { + header = {"edge types"}; + handler = [storage = current_db.db_acc_->get()->storage(), dba] { + auto edge_types = dba->ListAllPossiblyPresentEdgeTypes(); + std::vector> results; + results.reserve(edge_types.size()); + for (auto &edge_type : edge_types) { + results.push_back({TypedValue(edge_type)}); + } + + return std::pair{results, QueryHandlerResult::COMMIT}; + }; + + break; + } + case DatabaseInfoQuery::InfoType::NODE_LABELS: { + header = {"node labels"}; + handler = [storage = current_db.db_acc_->get()->storage(), dba] { + auto node_labels = dba->ListAllPossiblyPresentVertexLabels(); + std::vector> results; + results.reserve(node_labels.size()); + for (auto &node_label : node_labels) { + results.push_back({TypedValue(node_label)}); + } + + return std::pair{results, QueryHandlerResult::COMMIT}; + }; + + break; + } + + // NODE_LABELS } return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), From 260d60451d2b5edf22a62da9a1e3ee2a218c1b0e Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 7 Nov 2023 12:07:52 +0100 Subject: [PATCH 03/52] Modify retrieval function signatures Before the functions that are retrieving the data from the metadata holding datastructures were returning a std::string, and that was propagated outward all the way through. To keep this functions consistent with the rest of the storage/dbaccessor functions the LabelId and EdgeTypeId will be propagated respectively and the conversion into string will only happen at the interpreter level. --- src/query/db_accessor.hpp | 4 ++-- src/query/interpreter.cpp | 4 ++-- src/storage/v2/storage.cpp | 12 ++++++------ src/storage/v2/storage.hpp | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index 866da0af9..48c5f2131 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -597,10 +597,10 @@ class DbAccessor final { return accessor_->ApproximateVertexCount(label, property, lower, upper); } - std::vector ListAllPossiblyPresentVertexLabels() const { + std::vector ListAllPossiblyPresentVertexLabels() const { return accessor_->ListAllPossiblyPresentVertexLabels(); } - std::vector ListAllPossiblyPresentEdgeTypes() const { + std::vector ListAllPossiblyPresentEdgeTypes() const { return accessor_->ListAllPossiblyPresentEdgeTypes(); } diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 9313f4547..c3e2a9ac8 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -3083,7 +3083,7 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici std::vector> results; results.reserve(edge_types.size()); for (auto &edge_type : edge_types) { - results.push_back({TypedValue(edge_type)}); + results.push_back({TypedValue(storage->EdgeTypeToName(edge_type))}); } return std::pair{results, QueryHandlerResult::COMMIT}; @@ -3098,7 +3098,7 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici std::vector> results; results.reserve(node_labels.size()); for (auto &node_label : node_labels) { - results.push_back({TypedValue(node_label)}); + results.push_back({TypedValue(storage->LabelToName(node_label))}); } return std::pair{results, QueryHandlerResult::COMMIT}; diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 5b935c029..955b0868e 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -122,20 +122,20 @@ std::optional Storage::Accessor::GetTransactionId() const { return {}; } -std::vector Storage::Accessor::ListAllPossiblyPresentVertexLabels() const { - std::vector vertex_labels; +std::vector Storage::Accessor::ListAllPossiblyPresentVertexLabels() const { + std::vector vertex_labels; vertex_labels.reserve(storage_->stored_node_labels_.size()); for (const auto label : storage_->stored_node_labels_) { - vertex_labels.emplace_back(LabelToName(label)); + vertex_labels.push_back(label); } return vertex_labels; } -std::vector Storage::Accessor::ListAllPossiblyPresentEdgeTypes() const { - std::vector edge_types; +std::vector Storage::Accessor::ListAllPossiblyPresentEdgeTypes() const { + std::vector edge_types; edge_types.reserve(storage_->stored_edge_types_.size()); for (const auto edge_type : storage_->stored_edge_types_) { - edge_types.emplace_back(EdgeTypeToName(edge_type)); + edge_types.push_back(edge_type); } return edge_types; } diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index f3253267e..b45f12035 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -237,9 +237,9 @@ class Storage { const std::string &id() const { return storage_->id(); } - std::vector ListAllPossiblyPresentVertexLabels() const; + std::vector ListAllPossiblyPresentVertexLabels() const; - std::vector ListAllPossiblyPresentEdgeTypes() const; + std::vector ListAllPossiblyPresentEdgeTypes() const; virtual utils::BasicResult CreateIndex(LabelId label) = 0; From df3274d78fa615a9f63e731ba2e28b318166fcf2 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Wed, 8 Nov 2023 14:43:06 +0100 Subject: [PATCH 04/52] Make the metadata storing objects threadsafe The objects stored_node_labels_ and stored_edge_types_ can be accesses through separate threads but it was not safe to do so. This commit replaces the standard containers with threadsafe ones. --- src/storage/v2/disk/storage.cpp | 2 +- src/storage/v2/inmemory/storage.cpp | 4 +- src/storage/v2/storage.cpp | 10 +--- src/storage/v2/storage.hpp | 5 +- src/storage/v2/vertex_accessor.cpp | 2 +- src/utils/synchronized_metadata_store.hpp | 65 +++++++++++++++++++++++ 6 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 src/utils/synchronized_metadata_store.hpp diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 8dca66b29..ff9fc6f0c 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -944,7 +944,7 @@ Result DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT); transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN); - storage_->stored_edge_types_.insert(edge_type); + storage_->stored_edge_types_.try_insert(edge_type); storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel); return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_); diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 5ed493432..797ca844f 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -278,7 +278,7 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso if (to_vertex->deleted) return Error::DELETED_OBJECT; } - storage_->stored_edge_types_.insert(edge_type); + storage_->stored_edge_types_.try_insert(edge_type); auto *mem_storage = static_cast(storage_); auto gid = storage::Gid::FromUint(mem_storage->edge_id_.fetch_add(1, std::memory_order_acq_rel)); EdgeRef edge(gid); @@ -343,7 +343,7 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAcces if (to_vertex->deleted) return Error::DELETED_OBJECT; } - storage_->stored_edge_types_.insert(edge_type); + storage_->stored_edge_types_.try_insert(edge_type); // NOTE: When we update the next `edge_id_` here we perform a RMW // (read-modify-write) operation that ISN'T atomic! But, that isn't an issue diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 20d458e40..e07d96065 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -106,19 +106,13 @@ std::optional Storage::Accessor::GetTransactionId() const { std::vector Storage::Accessor::ListAllPossiblyPresentVertexLabels() const { std::vector vertex_labels; - vertex_labels.reserve(storage_->stored_node_labels_.size()); - for (const auto label : storage_->stored_node_labels_) { - vertex_labels.push_back(label); - } + storage_->stored_node_labels_.for_each([&vertex_labels](const auto &label) { vertex_labels.push_back(label); }); return vertex_labels; } std::vector Storage::Accessor::ListAllPossiblyPresentEdgeTypes() const { std::vector edge_types; - edge_types.reserve(storage_->stored_edge_types_.size()); - for (const auto edge_type : storage_->stored_edge_types_) { - edge_types.push_back(edge_type); - } + storage_->stored_edge_types_.for_each([&edge_types](const auto &type) { edge_types.push_back(type); }); return edge_types; } diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 0499b4665..b1bee1a6a 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -40,6 +40,7 @@ #include "utils/event_histogram.hpp" #include "utils/resource_lock.hpp" #include "utils/scheduler.hpp" +#include "utils/synchronized_metadata_store.hpp" #include "utils/timer.hpp" #include "utils/uuid.hpp" @@ -407,8 +408,8 @@ class Storage { // be present in the database. // TODO(gvolfing): check if this would be faster with flat_maps. - std::unordered_set stored_node_labels_; - std::unordered_set stored_edge_types_; + utils::SynchronizedMetaDataStore stored_node_labels_; + utils::SynchronizedMetaDataStore stored_edge_types_; std::atomic vertex_id_{0}; std::atomic edge_id_{0}; diff --git a/src/storage/v2/vertex_accessor.cpp b/src/storage/v2/vertex_accessor.cpp index 91ffd547e..a3527ff38 100644 --- a/src/storage/v2/vertex_accessor.cpp +++ b/src/storage/v2/vertex_accessor.cpp @@ -109,7 +109,7 @@ Result VertexAccessor::AddLabel(LabelId label) { CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label); vertex_->labels.push_back(label); - storage_->stored_node_labels_.insert(label); + storage_->stored_node_labels_.try_insert(label); /// TODO: some by pointers, some by reference => not good, make it better storage_->constraints_.unique_constraints_->UpdateOnAddLabel(label, *vertex_, transaction_->start_timestamp); diff --git a/src/utils/synchronized_metadata_store.hpp b/src/utils/synchronized_metadata_store.hpp new file mode 100644 index 000000000..0c0a85c21 --- /dev/null +++ b/src/utils/synchronized_metadata_store.hpp @@ -0,0 +1,65 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "utils/rw_lock.hpp" +#include "utils/synchronized.hpp" + +namespace memgraph::utils { + +template +class SynchronizedMetaDataStore { + public: + SynchronizedMetaDataStore() = default; + ~SynchronizedMetaDataStore() = default; + + SynchronizedMetaDataStore(const SynchronizedMetaDataStore &) = delete; + SynchronizedMetaDataStore(SynchronizedMetaDataStore &&) = delete; + SynchronizedMetaDataStore &operator=(const SynchronizedMetaDataStore &) = delete; + SynchronizedMetaDataStore &operator=(SynchronizedMetaDataStore &&) = delete; + + void try_insert(const T &elem) { + { + std::shared_lock read_lock(lock_); + if (element_store_.contains(elem)) { + return; + } + } + { + std::unique_lock write_lock(lock_); + element_store_.insert(elem); + } + } + + void erase(const T &elem) { + std::unique_lock write_lock(lock_); + element_store_.erase(elem); + } + + template + void for_each(const TFunc &func) { + std::unique_lock write_lock(lock_); + for (const auto &elem : element_store_) { + func(elem); + } + } + + private: + std::unordered_set element_store_; + RWLock lock_{RWLock::Priority::READ}; +}; + +} // namespace memgraph::utils From 3c413a7e50536a29416d9bd4853447e20780f0bf Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Sun, 12 Nov 2023 20:45:02 +0100 Subject: [PATCH 05/52] Fix hash join expression matching (#1496) --- src/query/plan/operator.cpp | 4 ++-- src/query/plan/rewrite/join.hpp | 8 ++++++++ .../tests/memgraph_V1/features/cartesian.feature | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 69748014e..792d278e8 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -5404,7 +5404,7 @@ class HashJoinCursor : public Cursor { // Check if the join value from the pulled frame is shared with any left frames ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, storage::View::OLD); - auto right_value = self_.hash_join_condition_->expression1_->Accept(evaluator); + auto right_value = self_.hash_join_condition_->expression2_->Accept(evaluator); if (hashtable_.contains(right_value)) { // If so, finish pulling for now and proceed to joining the pulled frame right_op_frame_.assign(frame.elems().begin(), frame.elems().end()); @@ -5452,7 +5452,7 @@ class HashJoinCursor : public Cursor { while (left_op_cursor_->Pull(frame, context)) { ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, storage::View::OLD); - auto left_value = self_.hash_join_condition_->expression2_->Accept(evaluator); + auto left_value = self_.hash_join_condition_->expression1_->Accept(evaluator); if (left_value.type() != TypedValue::Type::Null) { hashtable_[left_value].emplace_back(frame.elems().begin(), frame.elems().end()); } diff --git a/src/query/plan/rewrite/join.hpp b/src/query/plan/rewrite/join.hpp index c16f5b60d..5ed2bb090 100644 --- a/src/query/plan/rewrite/join.hpp +++ b/src/query/plan/rewrite/join.hpp @@ -27,6 +27,7 @@ #include "query/plan/operator.hpp" #include "query/plan/preprocess.hpp" +#include "utils/algorithm.hpp" namespace memgraph::query::plan { @@ -523,6 +524,13 @@ class JoinRewriter final : public HierarchicalLogicalOperatorVisitor { auto rhs_property = rhs_lookup->property_; filter_exprs_for_removal_.insert(filter.expression); filters_.EraseFilter(filter); + + if (utils::Contains(right_symbols, lhs_symbol) && utils::Contains(left_symbols, rhs_symbol)) { + // We need to duplicate this because expressions are shared between plans + join_condition = join_condition->Clone(ast_storage_); + std::swap(join_condition->expression1_, join_condition->expression2_); + } + return std::make_unique(left_op, left_symbols, right_op, right_symbols, join_condition); } diff --git a/tests/gql_behave/tests/memgraph_V1/features/cartesian.feature b/tests/gql_behave/tests/memgraph_V1/features/cartesian.feature index 809a3d73a..b11d83f0e 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/cartesian.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/cartesian.feature @@ -186,6 +186,20 @@ Feature: Cartesian | a | b | | (:A {id: 1}) | (:B {id: 1}) | + Scenario: Multiple match with WHERE x = y 01 reversed + Given an empty graph + And having executed + """ + CREATE (:A {id: 1}), (:A {id: 2}), (:B {id: 1}) + """ + When executing query: + """ + MATCH (a:A) MATCH (b:B) WHERE b.id = a.id RETURN a, b + """ + Then the result should be: + | a | b | + | (:A {id: 1}) | (:B {id: 1}) | + Scenario: Multiple match with WHERE x = y 02 Given an empty graph And having executed From 38ad5e2146a2e35f6bd28fdb5d6c219bc9e5e1df Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Sun, 12 Nov 2023 23:51:00 +0100 Subject: [PATCH 06/52] Fix parallel index loading (#1479) --- src/storage/v2/durability/durability.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index a3cf0e2bb..decbfd14a 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -397,7 +397,12 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di spdlog::info("All necessary WAL files are loaded successfully."); } - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices); + const auto par_exec_info = + config.durability.allow_parallel_index_creation && !recovery_info.vertex_batches.empty() + ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) + : std::nullopt; + + RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info); memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us, std::chrono::duration_cast(timer.Elapsed()).count()); From 0756cd6898cd5ca06e51049781b644a0b5f6f8d1 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Mon, 13 Nov 2023 04:12:25 +0100 Subject: [PATCH 07/52] Add fix indexed join crash (#1478) --- src/query/plan/rewrite/join.hpp | 5 +++- tests/e2e/CMakeLists.txt | 1 + tests/e2e/queries/CMakeLists.txt | 6 +++++ tests/e2e/queries/common.py | 24 ++++++++++++++++++++ tests/e2e/queries/queries.py | 39 ++++++++++++++++++++++++++++++++ tests/e2e/queries/workloads.yaml | 14 ++++++++++++ 6 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/queries/CMakeLists.txt create mode 100644 tests/e2e/queries/common.py create mode 100644 tests/e2e/queries/queries.py create mode 100644 tests/e2e/queries/workloads.yaml diff --git a/src/query/plan/rewrite/join.hpp b/src/query/plan/rewrite/join.hpp index 5ed2bb090..65e32b3e8 100644 --- a/src/query/plan/rewrite/join.hpp +++ b/src/query/plan/rewrite/join.hpp @@ -146,8 +146,11 @@ class JoinRewriter final : public HierarchicalLogicalOperatorVisitor { bool PreVisit(IndexedJoin &op) override { prev_ops_.push_back(&op); - return true; + RewriteBranch(&op.main_branch_); + RewriteBranch(&op.sub_branch_); + return false; } + bool PostVisit(IndexedJoin &) override { prev_ops_.pop_back(); return true; diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 7c7edb60e..71d80b7ed 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -70,6 +70,7 @@ add_subdirectory(index_hints) add_subdirectory(query_modules) add_subdirectory(constraints) add_subdirectory(inspect_query) +add_subdirectory(queries) copy_e2e_python_files(pytest_runner pytest_runner.sh "") copy_e2e_python_files(x x.sh "") diff --git a/tests/e2e/queries/CMakeLists.txt b/tests/e2e/queries/CMakeLists.txt new file mode 100644 index 000000000..f672b8591 --- /dev/null +++ b/tests/e2e/queries/CMakeLists.txt @@ -0,0 +1,6 @@ +function(copy_queries_e2e_python_files FILE_NAME) + copy_e2e_python_files(queries ${FILE_NAME}) +endfunction() + +copy_queries_e2e_python_files(common.py) +copy_queries_e2e_python_files(queries.py) diff --git a/tests/e2e/queries/common.py b/tests/e2e/queries/common.py new file mode 100644 index 000000000..6ad52539b --- /dev/null +++ b/tests/e2e/queries/common.py @@ -0,0 +1,24 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import pytest +from gqlalchemy import Memgraph + + +@pytest.fixture +def memgraph(**kwargs) -> Memgraph: + memgraph = Memgraph() + + yield memgraph + + memgraph.drop_indexes() + memgraph.ensure_constraints([]) + memgraph.drop_database() diff --git a/tests/e2e/queries/queries.py b/tests/e2e/queries/queries.py new file mode 100644 index 000000000..61cea625f --- /dev/null +++ b/tests/e2e/queries/queries.py @@ -0,0 +1,39 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys + +import pytest +from common import memgraph + + +def test_indexed_join_with_indices(memgraph): + memgraph.execute( + "CREATE (c:A {prop: 1})-[b:TYPE]->(p:A {prop: 1}) CREATE (cf:B:A {prop : 1}) CREATE (pf:B:A {prop : 1});" + ) + memgraph.execute("CREATE INDEX ON :A;") + memgraph.execute("CREATE INDEX ON :B;") + memgraph.execute("CREATE INDEX ON :A(prop);") + memgraph.execute("CREATE INDEX ON :B(prop);") + + results = list( + memgraph.execute_and_fetch( + "match (c:A)-[b:TYPE]->(p:A) match (cf:B:A {prop : c.prop}) match (pf:B:A {prop : p.prop}) return c;" + ) + ) + + assert len(results) == 4 + for res in results: + assert res["c"].prop == 1 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/queries/workloads.yaml b/tests/e2e/queries/workloads.yaml new file mode 100644 index 000000000..cb9ac4d09 --- /dev/null +++ b/tests/e2e/queries/workloads.yaml @@ -0,0 +1,14 @@ +queries_cluster: &queries_cluster + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "queries.log" + setup_queries: [] + validation_queries: [] + + +workloads: + - name: "Queries validation" + binary: "tests/e2e/pytest_runner.sh" + args: ["queries/queries.py"] + <<: *queries_cluster From e90781785412cc07792b1150addaba393206a3cc Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Mon, 13 Nov 2023 05:17:10 +0100 Subject: [PATCH 08/52] Fix for in list segmentation fault (#1494) --- src/query/frame_change.hpp | 17 ----------------- src/query/interpret/eval.hpp | 5 ++--- src/query/interpreter.cpp | 1 - .../features/list_operations.feature | 16 ++++++++++++++++ 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/query/frame_change.hpp b/src/query/frame_change.hpp index 535d5ddb9..1d9ebc70c 100644 --- a/src/query/frame_change.hpp +++ b/src/query/frame_change.hpp @@ -43,23 +43,6 @@ struct CachedValue { return cache_.get_allocator().GetMemoryResource(); } - // Func to check if cache_ contains value - bool CacheValue(TypedValue &&maybe_list) { - if (!maybe_list.IsList()) { - return false; - } - auto &list = maybe_list.ValueList(); - TypedValue::Hash hash{}; - for (auto &element : list) { - const auto key = hash(element); - auto &vector_values = cache_[key]; - if (!IsValueInVec(vector_values, element)) { - vector_values.emplace_back(std::move(element)); - } - } - return true; - } - bool CacheValue(const TypedValue &maybe_list) { if (!maybe_list.IsList()) { return false; diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index f9ebf5467..333a7b1fa 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -315,8 +315,8 @@ class ExpressionEvaluator : public ExpressionVisitor { return std::move(*preoperational_checks); } auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id); - cached_value.CacheValue(std::move(list)); - spdlog::trace("Value cached {}", *cached_id); + // Don't move here because we don't want to remove the element from the frame + cached_value.CacheValue(list); } const auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id); @@ -338,7 +338,6 @@ class ExpressionEvaluator : public ExpressionVisitor { } const auto &list_value = list.ValueList(); - spdlog::trace("Not using cache on IN LIST operator"); auto has_null = false; for (const auto &element : list_value) { auto result = literal == element; diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 2e5bba2ff..30bb4eca2 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -1535,7 +1535,6 @@ inline static void TryCaching(const AstStorage &ast_storage, FrameChangeCollecto continue; } frame_change_collector->AddTrackingKey(*cached_id); - spdlog::trace("Tracking {} operator, by id: {}", InListOperator::kType.name, *cached_id); } } diff --git a/tests/gql_behave/tests/memgraph_V1/features/list_operations.feature b/tests/gql_behave/tests/memgraph_V1/features/list_operations.feature index eed738446..bfe6b6225 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/list_operations.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/list_operations.feature @@ -263,3 +263,19 @@ Feature: List operators | id | | 1 | | 2 | + + Scenario: InList 01 + Given an empty graph + And having executed + """ + CREATE (o:Node) SET o.Status = 'This is the status'; + """ + When executing query: + """ + match (o:Node) + where o.Status IN ['This is not the status', 'This is the status'] + return o; + """ + Then the result should be: + | o | + | (:Node {Status: 'This is the status'}) | From e5b2c19ea2560af4995af7831648bbdb80abc0a1 Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 13 Nov 2023 11:45:09 +0100 Subject: [PATCH 09/52] Empty Collect() returns nothing (#1482) --- src/query/plan/operator.cpp | 16 +++++++++-- src/query/plan/operator.hpp | 3 ++ .../show_while_creating_invalid_state.py | 2 +- .../memgraph_V1/features/aggregations.feature | 28 ++++++++++++++++++- .../features/aggregations.feature | 28 ++++++++++++++++++- .../unit/query_plan_accumulate_aggregate.cpp | 12 +++----- 6 files changed, 75 insertions(+), 14 deletions(-) diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 792d278e8..238638737 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -3463,7 +3463,7 @@ class AggregateCursor : public Cursor { SCOPED_PROFILE_OP_BY_REF(self_); if (!pulled_all_input_) { - ProcessAll(&frame, &context); + if (!ProcessAll(&frame, &context) && self_.AreAllAggregationsForCollecting()) return false; pulled_all_input_ = true; aggregation_it_ = aggregation_.begin(); @@ -3487,7 +3487,6 @@ class AggregateCursor : public Cursor { return true; } } - if (aggregation_it_ == aggregation_.end()) return false; // place aggregation values on the frame @@ -3567,12 +3566,16 @@ class AggregateCursor : public Cursor { * cache cardinality depends on number of * aggregation results, and not on the number of inputs. */ - void ProcessAll(Frame *frame, ExecutionContext *context) { + bool ProcessAll(Frame *frame, ExecutionContext *context) { ExpressionEvaluator evaluator(frame, context->symbol_table, context->evaluation_context, context->db_accessor, storage::View::NEW); + + bool pulled = false; while (input_cursor_->Pull(*frame, *context)) { ProcessOne(*frame, &evaluator); + pulled = true; } + if (!pulled) return false; // post processing for (size_t pos = 0; pos < self_.aggregations_.size(); ++pos) { @@ -3606,6 +3609,7 @@ class AggregateCursor : public Cursor { break; } } + return true; } /** @@ -3819,6 +3823,12 @@ UniqueCursorPtr Aggregate::MakeCursor(utils::MemoryResource *mem) const { return MakeUniqueCursorPtr(mem, *this, mem); } +auto Aggregate::AreAllAggregationsForCollecting() const -> bool { + return std::all_of(aggregations_.begin(), aggregations_.end(), [](const auto &agg) { + return agg.op == Aggregation::Op::COLLECT_LIST || agg.op == Aggregation::Op::COLLECT_MAP; + }); +} + Skip::Skip(const std::shared_ptr &input, Expression *expression) : input_(input), expression_(expression) {} diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index 4951b5137..ba844796a 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -1758,6 +1758,9 @@ class Aggregate : public memgraph::query::plan::LogicalOperator { Aggregate() = default; Aggregate(const std::shared_ptr &input, const std::vector &aggregations, const std::vector &group_by, const std::vector &remember); + + auto AreAllAggregationsForCollecting() const -> bool; + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; diff --git a/tests/e2e/replication/show_while_creating_invalid_state.py b/tests/e2e/replication/show_while_creating_invalid_state.py index f8fae4cd6..996955dc1 100644 --- a/tests/e2e/replication/show_while_creating_invalid_state.py +++ b/tests/e2e/replication/show_while_creating_invalid_state.py @@ -697,7 +697,7 @@ def test_sync_replication_when_main_is_killed(): ) # 2/ - QUERY_TO_CHECK = "MATCH (n) RETURN COLLECT(n.name);" + QUERY_TO_CHECK = "MATCH (n) RETURN COUNT(n.name);" last_result_from_main = interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(QUERY_TO_CHECK)[0][0] for index in range(50): interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(f"CREATE (p:Number {{name:{index}}})") diff --git a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature index 8fe6a47ad..cff138432 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature @@ -401,4 +401,30 @@ Feature: Aggregations MATCH p=()-[:Z]->() WITH project(p) as graph WITH graph.edges as edges UNWIND edges as e RETURN e.prop as y ORDER BY y DESC """ Then the result should be: - | y | + | y | + + Scenario: Empty collect aggregation: + Given an empty graph + And having executed + """ + CREATE (s:Subnet {ip: "192.168.0.1"}) + """ + When executing query: + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id + """ + Then the result should be empty + + Scenario: Empty count aggregation: + Given an empty graph + And having executed + """ + CREATE (s:Subnet {ip: "192.168.0.1"}) + """ + When executing query: + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id + """ + Then the result should be: + | id | + | null | diff --git a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature index 8fe6a47ad..cff138432 100644 --- a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature @@ -401,4 +401,30 @@ Feature: Aggregations MATCH p=()-[:Z]->() WITH project(p) as graph WITH graph.edges as edges UNWIND edges as e RETURN e.prop as y ORDER BY y DESC """ Then the result should be: - | y | + | y | + + Scenario: Empty collect aggregation: + Given an empty graph + And having executed + """ + CREATE (s:Subnet {ip: "192.168.0.1"}) + """ + When executing query: + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id + """ + Then the result should be empty + + Scenario: Empty count aggregation: + Given an empty graph + And having executed + """ + CREATE (s:Subnet {ip: "192.168.0.1"}) + """ + When executing query: + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id + """ + Then the result should be: + | id | + | null | diff --git a/tests/unit/query_plan_accumulate_aggregate.cpp b/tests/unit/query_plan_accumulate_aggregate.cpp index e271e0f6a..bbf3e0311 100644 --- a/tests/unit/query_plan_accumulate_aggregate.cpp +++ b/tests/unit/query_plan_accumulate_aggregate.cpp @@ -277,13 +277,11 @@ TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithGroupBy) { } { auto results = this->AggregationResults(true, false, {Aggregation::Op::COLLECT_LIST}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::List); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::COLLECT_MAP}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Map); + EXPECT_EQ(results.size(), 0); } } @@ -695,13 +693,11 @@ TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithDistinctAndWithGroupBy) { } { auto results = this->AggregationResults(true, true, {Aggregation::Op::COLLECT_LIST}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::List); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::COLLECT_MAP}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Map); + EXPECT_EQ(results.size(), 0); } } From fdab42a023840317622aacaad898dcd55177ffda Mon Sep 17 00:00:00 2001 From: DavIvek Date: Mon, 13 Nov 2023 12:08:48 +0100 Subject: [PATCH 10/52] Use static linking on c++ query modules for glibcxx (#1490) --- query_modules/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/query_modules/CMakeLists.txt b/query_modules/CMakeLists.txt index 879879e89..41dbb495c 100644 --- a/query_modules/CMakeLists.txt +++ b/query_modules/CMakeLists.txt @@ -13,6 +13,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) add_library(example_c SHARED example.c) target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include) target_compile_options(example_c PRIVATE -Wall) +target_link_libraries(example_c PRIVATE -static-libgcc -static-libstdc++) # Strip C example in release build. if (lower_build_type STREQUAL "release") add_custom_command(TARGET example_c POST_BUILD @@ -28,6 +29,7 @@ install(FILES example.c DESTINATION lib/memgraph/query_modules/src) add_library(example_cpp SHARED example.cpp) target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include) target_compile_options(example_cpp PRIVATE -Wall) +target_link_libraries(example_cpp PRIVATE -static-libgcc -static-libstdc++) # Strip C++ example in release build. if (lower_build_type STREQUAL "release") add_custom_command(TARGET example_cpp POST_BUILD @@ -43,6 +45,7 @@ install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src) add_library(schema SHARED schema.cpp) target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include) target_compile_options(schema PRIVATE -Wall) +target_link_libraries(schema PRIVATE -static-libgcc -static-libstdc++) # Strip C++ example in release build. if (lower_build_type STREQUAL "release") add_custom_command(TARGET schema POST_BUILD From 11be3972c40d0d7c72a6fe17e9739e0c662d5e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:20:49 +0100 Subject: [PATCH 11/52] Add workflow for packaging memgraph for specific target OS (#1502) --- .github/workflows/package_specific.yaml | 315 ++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 .github/workflows/package_specific.yaml diff --git a/.github/workflows/package_specific.yaml b/.github/workflows/package_specific.yaml new file mode 100644 index 000000000..6181524db --- /dev/null +++ b/.github/workflows/package_specific.yaml @@ -0,0 +1,315 @@ +name: Package Specific + +# TODO(gitbuda): Cleanup docker container if GHA job was canceled. + +on: + workflow_dispatch: + inputs: + memgraph_version: + description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'" + required: false + build_type: + type: choice + description: "Memgraph Build type. Default value is Release." + default: 'Release' + options: + - Release + - RelWithDebInfo + build_amzn_2: + type: boolean + default: false + build_centos_7: + type: boolean + default: false + build_centos_9: + type: boolean + default: false + build_debian_10: + type: boolean + default: false + build_debian_11: + type: boolean + default: false + build_debian_11_arm: + type: boolean + default: false + build_debian_11_platform: + type: boolean + default: false + build_docker: + type: boolean + default: false + build_fedora_36: + type: boolean + default: false + build_ubuntu_18_04: + type: boolean + default: false + build_ubuntu_20_04: + type: boolean + default: false + build_ubuntu_22_04: + type: boolean + default: false + build_ubuntu_22_04_arm: + type: boolean + default: false + +jobs: + amzn-2: + if: ${{ github.event.inputs.build_amzn_2 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: amzn-2 + path: build/output/amzn-2/memgraph*.rpm + + centos-7: + if: ${{ github.event.inputs.build_centos_7 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: centos-7 + path: build/output/centos-7/memgraph*.rpm + + centos-9: + if: ${{ github.event.inputs.build_centos_9 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: centos-9 + path: build/output/centos-9/memgraph*.rpm + + debian-10: + if: ${{ github.event.inputs.build_debian_10 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: debian-10 + path: build/output/debian-10/memgraph*.deb + + debian-11: + if: ${{ github.event.inputs.build_debian_11 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: debian-11 + path: build/output/debian-11/memgraph*.deb + + debian-11-arm: + if: ${{ github.event.inputs.build_debian_11_arm }} + runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + timeout-minutes: 120 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: debian-11-aarch64 + path: build/output/debian-11-arm/memgraph*.deb + + debian-11-platform: + if: ${{ github.event.inputs.build_debian_11_platform }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: debian-11-platform + path: build/output/debian-11/memgraph*.deb + + docker: + if: ${{ github.event.inputs.build_docker }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + cd release/package + ./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker + ./run.sh docker + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: docker + path: build/output/docker/memgraph*.tar.gz + + fedora-36: + if: ${{ github.event.inputs.build_fedora_36 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: fedora-36 + path: build/output/fedora-36/memgraph*.rpm + + ubuntu-18_04: + if: ${{ github.event.inputs.build_ubuntu_18_04 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: ubuntu-18.04 + path: build/output/ubuntu-18.04/memgraph*.deb + + ubuntu-20_04: + if: ${{ github.event.inputs.build_ubuntu_20_04 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: ubuntu-20.04 + path: build/output/ubuntu-20.04/memgraph*.deb + + ubuntu-22_04: + if: ${{ github.event.inputs.build_ubuntu_22_04 }} + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: ubuntu-22.04 + path: build/output/ubuntu-22.04/memgraph*.deb + + ubuntu-22_04-arm: + if: ${{ github.event.inputs.build_ubuntu_22_04_arm }} + runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + timeout-minutes: 120 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }} + - name: "Upload package" + uses: actions/upload-artifact@v3 + with: + name: ubuntu-22.04-aarch64 + path: build/output/ubuntu-22.04-arm/memgraph*.deb + + upload-to-s3: + # only run upload if we specified version. Allows for runs without upload + if: "${{ github.event.inputs.memgraph_version != '' }}" + needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm] + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + # name: # if name input parameter is not provided, all artifacts are downloaded + # and put in directories named after each one. + path: build/output/release + - name: Upload to S3 + uses: jakejarvis/s3-sync-action@v0.5.1 + env: + AWS_S3_BUCKET: "download.memgraph.com" + AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "eu-west-1" + SOURCE_DIR: "build/output/release" + DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/" From e671a0737e5826fec1a823da8dda9a3f7331fb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:54:19 +0100 Subject: [PATCH 12/52] Fix package specific workflow file (#1503) --- .github/workflows/package_specific.yaml | 83 +++++++++---------------- 1 file changed, 31 insertions(+), 52 deletions(-) diff --git a/.github/workflows/package_specific.yaml b/.github/workflows/package_specific.yaml index 6181524db..2d3d26ef5 100644 --- a/.github/workflows/package_specific.yaml +++ b/.github/workflows/package_specific.yaml @@ -15,49 +15,28 @@ on: options: - Release - RelWithDebInfo - build_amzn_2: - type: boolean - default: false - build_centos_7: - type: boolean - default: false - build_centos_9: - type: boolean - default: false - build_debian_10: - type: boolean - default: false - build_debian_11: - type: boolean - default: false - build_debian_11_arm: - type: boolean - default: false - build_debian_11_platform: - type: boolean - default: false - build_docker: - type: boolean - default: false - build_fedora_36: - type: boolean - default: false - build_ubuntu_18_04: - type: boolean - default: false - build_ubuntu_20_04: - type: boolean - default: false - build_ubuntu_22_04: - type: boolean - default: false - build_ubuntu_22_04_arm: - type: boolean - default: false + target_os: + type: choice + description: "Target OS for which memgraph will be packaged. Default is Ubuntu 22.04" + default: 'ubuntu-22_04' + options: + - amzn-2 + - centos-7 + - centos-9 + - debian-10 + - debian-11 + - debian-11-arm + - debian-11-platform + - docker + - fedora-36 + - ubuntu-18_04 + - ubuntu-20_04 + - ubuntu-22_04 + - ubuntu-22_04-arm jobs: amzn-2: - if: ${{ github.event.inputs.build_amzn_2 }} + if: ${{ github.event.inputs.target_os == 'amzn-2' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -75,7 +54,7 @@ jobs: path: build/output/amzn-2/memgraph*.rpm centos-7: - if: ${{ github.event.inputs.build_centos_7 }} + if: ${{ github.event.inputs.target_os == 'centos-7' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -93,7 +72,7 @@ jobs: path: build/output/centos-7/memgraph*.rpm centos-9: - if: ${{ github.event.inputs.build_centos_9 }} + if: ${{ github.event.inputs.target_os == 'centos-9' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -111,7 +90,7 @@ jobs: path: build/output/centos-9/memgraph*.rpm debian-10: - if: ${{ github.event.inputs.build_debian_10 }} + if: ${{ github.event.inputs.target_os == 'debian-10' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -129,7 +108,7 @@ jobs: path: build/output/debian-10/memgraph*.deb debian-11: - if: ${{ github.event.inputs.build_debian_11 }} + if: ${{ github.event.inputs.target_os == 'debian-11' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -147,7 +126,7 @@ jobs: path: build/output/debian-11/memgraph*.deb debian-11-arm: - if: ${{ github.event.inputs.build_debian_11_arm }} + if: ${{ github.event.inputs.target_os == 'debian-11-arm' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: @@ -165,7 +144,7 @@ jobs: path: build/output/debian-11-arm/memgraph*.deb debian-11-platform: - if: ${{ github.event.inputs.build_debian_11_platform }} + if: ${{ github.event.inputs.target_os == 'debian-11-platform' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -183,7 +162,7 @@ jobs: path: build/output/debian-11/memgraph*.deb docker: - if: ${{ github.event.inputs.build_docker }} + if: ${{ github.event.inputs.target_os == 'docker' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -203,7 +182,7 @@ jobs: path: build/output/docker/memgraph*.tar.gz fedora-36: - if: ${{ github.event.inputs.build_fedora_36 }} + if: ${{ github.event.inputs.target_os == 'fedora-36' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -221,7 +200,7 @@ jobs: path: build/output/fedora-36/memgraph*.rpm ubuntu-18_04: - if: ${{ github.event.inputs.build_ubuntu_18_04 }} + if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -239,7 +218,7 @@ jobs: path: build/output/ubuntu-18.04/memgraph*.deb ubuntu-20_04: - if: ${{ github.event.inputs.build_ubuntu_20_04 }} + if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -257,7 +236,7 @@ jobs: path: build/output/ubuntu-20.04/memgraph*.deb ubuntu-22_04: - if: ${{ github.event.inputs.build_ubuntu_22_04 }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -275,7 +254,7 @@ jobs: path: build/output/ubuntu-22.04/memgraph*.deb ubuntu-22_04-arm: - if: ${{ github.event.inputs.build_ubuntu_22_04_arm }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: From 9cc060c4b0703a3fb6e9b0935bf8db584d0ef58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Mon, 13 Nov 2023 13:01:01 +0100 Subject: [PATCH 13/52] Fix error in upload-to-s3 job (#1504) --- .github/workflows/package_specific.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package_specific.yaml b/.github/workflows/package_specific.yaml index 2d3d26ef5..c599f65ef 100644 --- a/.github/workflows/package_specific.yaml +++ b/.github/workflows/package_specific.yaml @@ -274,7 +274,7 @@ jobs: upload-to-s3: # only run upload if we specified version. Allows for runs without upload if: "${{ github.event.inputs.memgraph_version != '' }}" - needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm] + needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm] runs-on: ubuntu-latest steps: - name: Download artifacts From 1527bdf43526669bea284d83f9fe2a3fcd0c9ac4 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 14 Nov 2023 13:10:08 +0100 Subject: [PATCH 14/52] Make metadata collection setable with flag There might be a performance impect of updating the metadata store on bulk operations. Hence this flag which is disabling the collection by default. If the queries to obtain the information are called with this flag disabled, the database will throw an exception. --- config/flags.yaml | 4 ++++ src/flags/general.cpp | 4 ++++ src/flags/general.hpp | 2 ++ src/memgraph.cpp | 3 ++- src/query/interpreter.cpp | 10 ++++++++++ src/storage/v2/config.hpp | 1 + src/storage/v2/disk/storage.cpp | 4 +++- src/storage/v2/inmemory/storage.cpp | 8 ++++++-- src/storage/v2/vertex_accessor.cpp | 5 ++++- tests/e2e/configuration/default_config.py | 5 +++++ 10 files changed, 41 insertions(+), 5 deletions(-) diff --git a/config/flags.yaml b/config/flags.yaml index 621d27345..cd3eee160 100644 --- a/config/flags.yaml +++ b/config/flags.yaml @@ -111,6 +111,10 @@ modifications: value: "false" override: true + - name: "storage_enable_schema_metadata" + value: "false" + override: true + - name: "query_callable_mappings_path" value: "/etc/memgraph/apoc_compatibility_mappings.json" override: true diff --git a/src/flags/general.cpp b/src/flags/general.cpp index eb8fae589..a50f83c25 100644 --- a/src/flags/general.cpp +++ b/src/flags/general.cpp @@ -114,6 +114,10 @@ DEFINE_uint64(storage_recovery_thread_count, memgraph::storage::Config::Durability().recovery_thread_count), "The number of threads used to recover persisted data from disk."); +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +DEFINE_bool(storage_enable_schema_metadata, false, + "Controls whether metadata should be collected about the resident labels and edge types."); + #ifdef MG_ENTERPRISE // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DEFINE_bool(storage_delete_on_drop, true, diff --git a/src/flags/general.hpp b/src/flags/general.hpp index f9b8f3517..eba59228b 100644 --- a/src/flags/general.hpp +++ b/src/flags/general.hpp @@ -77,6 +77,8 @@ DECLARE_uint64(storage_items_per_batch); DECLARE_bool(storage_parallel_index_recovery); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_uint64(storage_recovery_thread_count); +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +DECLARE_bool(storage_enable_schema_metadata); #ifdef MG_ENTERPRISE // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_bool(storage_delete_on_drop); diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 983dd61f9..0f6cd6a0c 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -291,7 +291,8 @@ int main(int argc, char **argv) { memgraph::storage::Config db_config{ .gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC, .interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)}, - .items = {.properties_on_edges = FLAGS_storage_properties_on_edges}, + .items = {.properties_on_edges = FLAGS_storage_properties_on_edges, + .enable_schema_metadata = FLAGS_storage_enable_schema_metadata}, .durability = {.storage_directory = FLAGS_data_directory, .recover_on_startup = FLAGS_storage_recover_on_startup || FLAGS_data_recovery_on_startup, .snapshot_retention_count = FLAGS_storage_snapshot_retention_count, diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index bc0bc93f3..62b6e690d 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -3049,6 +3049,11 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici case DatabaseInfoQuery::InfoType::EDGE_TYPES: { header = {"edge types"}; handler = [storage = current_db.db_acc_->get()->storage(), dba] { + if (!storage->config_.items.enable_schema_metadata) { + throw QueryRuntimeException( + "The metadata collection for edge-types is disabled. To enable it, restart your instance and set the " + "storage-enable-schema-metadata flag to True."); + } auto edge_types = dba->ListAllPossiblyPresentEdgeTypes(); std::vector> results; results.reserve(edge_types.size()); @@ -3064,6 +3069,11 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici case DatabaseInfoQuery::InfoType::NODE_LABELS: { header = {"node labels"}; handler = [storage = current_db.db_acc_->get()->storage(), dba] { + if (!storage->config_.items.enable_schema_metadata) { + throw QueryRuntimeException( + "The metadata collection for node-labels is disabled. To enable it, restart your instance and set the " + "storage-enable-schema-metadata flag to True."); + } auto node_labels = dba->ListAllPossiblyPresentVertexLabels(); std::vector> results; results.reserve(node_labels.size()); diff --git a/src/storage/v2/config.hpp b/src/storage/v2/config.hpp index 7ea7e95b7..4f333b5d9 100644 --- a/src/storage/v2/config.hpp +++ b/src/storage/v2/config.hpp @@ -40,6 +40,7 @@ struct Config { struct Items { bool properties_on_edges{true}; + bool enable_schema_metadata{false}; friend bool operator==(const Items &lrh, const Items &rhs) = default; } items; diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index ff9fc6f0c..87a0025ed 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -944,7 +944,9 @@ Result DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT); transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN); - storage_->stored_edge_types_.try_insert(edge_type); + if (storage_->config_.items.enable_schema_metadata) { + storage_->stored_edge_types_.try_insert(edge_type); + } storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel); return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_); diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 797ca844f..a342a34db 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -278,7 +278,9 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso if (to_vertex->deleted) return Error::DELETED_OBJECT; } - storage_->stored_edge_types_.try_insert(edge_type); + if (storage_->config_.items.enable_schema_metadata) { + storage_->stored_edge_types_.try_insert(edge_type); + } auto *mem_storage = static_cast(storage_); auto gid = storage::Gid::FromUint(mem_storage->edge_id_.fetch_add(1, std::memory_order_acq_rel)); EdgeRef edge(gid); @@ -343,7 +345,9 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAcces if (to_vertex->deleted) return Error::DELETED_OBJECT; } - storage_->stored_edge_types_.try_insert(edge_type); + if (storage_->config_.items.enable_schema_metadata) { + storage_->stored_edge_types_.try_insert(edge_type); + } // NOTE: When we update the next `edge_id_` here we perform a RMW // (read-modify-write) operation that ISN'T atomic! But, that isn't an issue diff --git a/src/storage/v2/vertex_accessor.cpp b/src/storage/v2/vertex_accessor.cpp index a3527ff38..ff5881563 100644 --- a/src/storage/v2/vertex_accessor.cpp +++ b/src/storage/v2/vertex_accessor.cpp @@ -109,7 +109,10 @@ Result VertexAccessor::AddLabel(LabelId label) { CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label); vertex_->labels.push_back(label); - storage_->stored_node_labels_.try_insert(label); + + if (storage_->config_.items.enable_schema_metadata) { + storage_->stored_node_labels_.try_insert(label); + } /// TODO: some by pointers, some by reference => not good, make it better storage_->constraints_.unique_constraints_->UpdateOnAddLabel(label, *vertex_, transaction_->start_timestamp); diff --git a/tests/e2e/configuration/default_config.py b/tests/e2e/configuration/default_config.py index 9a251c15c..025ba4d0a 100644 --- a/tests/e2e/configuration/default_config.py +++ b/tests/e2e/configuration/default_config.py @@ -115,6 +115,11 @@ startup_config_dict = { "false", "Controls whether the index creation can be done in a multithreaded fashion.", ), + "storage_enable_schema_metadata": ( + "false", + "false", + "Controls whether metadata should be collected about the resident labels and edge types.", + ), "password_encryption_algorithm": ("bcrypt", "bcrypt", "The password encryption algorithm used for authentication."), "pulsar_service_url": ("", "", "Default URL used while connecting to Pulsar brokers."), "query_execution_timeout_sec": ( From ced08fd7bc386a72ba6839cee7a7781e353faaf3 Mon Sep 17 00:00:00 2001 From: imilinovic <44698587+imilinovic@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:06:21 +0100 Subject: [PATCH 15/52] Fix GC by adding periodic jemalloc purge (#1471) --- CMakeLists.txt | 2 +- src/flags/general.cpp | 2 +- src/memory/CMakeLists.txt | 8 +-- src/storage/v2/CMakeLists.txt | 2 +- src/storage/v2/inmemory/storage.cpp | 11 +++- src/storage/v2/inmemory/storage.hpp | 1 + tests/e2e/CMakeLists.txt | 1 + tests/e2e/garbage_collection/CMakeLists.txt | 7 +++ tests/e2e/garbage_collection/common.py | 25 +++++++++ tests/e2e/garbage_collection/conftest.py | 21 ++++++++ tests/e2e/garbage_collection/gc_periodic.py | 59 +++++++++++++++++++++ tests/e2e/garbage_collection/workloads.yaml | 19 +++++++ 12 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/garbage_collection/CMakeLists.txt create mode 100644 tests/e2e/garbage_collection/common.py create mode 100644 tests/e2e/garbage_collection/conftest.py create mode 100644 tests/e2e/garbage_collection/gc_periodic.py create mode 100644 tests/e2e/garbage_collection/workloads.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index db39a4547..a5ad2612a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,7 +292,7 @@ if (MG_ENTERPRISE) add_definitions(-DMG_ENTERPRISE) endif() -set(ENABLE_JEMALLOC ON) +option(ENABLE_JEMALLOC "Use jemalloc" ON) if (ASAN) message(WARNING "Disabling jemalloc as it doesn't work well with ASAN") diff --git a/src/flags/general.cpp b/src/flags/general.cpp index eb8fae589..be060c52d 100644 --- a/src/flags/general.cpp +++ b/src/flags/general.cpp @@ -65,7 +65,7 @@ DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed i // Storage flags. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).", - FLAG_IN_RANGE(1, 24 * 3600)); + FLAG_IN_RANGE(1, 24UL * 3600)); // NOTE: The `storage_properties_on_edges` flag must be the same here and in // `mg_import_csv`. If you change it, make sure to change it there as well. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/memory/CMakeLists.txt b/src/memory/CMakeLists.txt index aadbbe23c..e975c1d5c 100644 --- a/src/memory/CMakeLists.txt +++ b/src/memory/CMakeLists.txt @@ -3,14 +3,14 @@ set(memory_src_files global_memory_control.cpp query_memory_control.cpp) - - -find_package(jemalloc REQUIRED) - add_library(mg-memory STATIC ${memory_src_files}) target_link_libraries(mg-memory mg-utils fmt) +message(STATUS "ENABLE_JEMALLOC: ${ENABLE_JEMALLOC}") if (ENABLE_JEMALLOC) + find_package(jemalloc REQUIRED) target_link_libraries(mg-memory Jemalloc::Jemalloc ${CMAKE_DL_LIBS}) target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1) +else() + target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=0) endif() diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 147684c54..9f6d8d4d7 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -41,4 +41,4 @@ add_library(mg-storage-v2 STATIC replication/replication_storage_state.cpp inmemory/replication/replication_client.cpp ) -target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events) +target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory) diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 9d2bc7a65..d0d1dd071 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -11,6 +11,7 @@ #include "storage/v2/inmemory/storage.hpp" #include "dbms/constants.hpp" +#include "memory/global_memory_control.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/metadata_delta.hpp" @@ -101,8 +102,13 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) "those files into a .backup directory inside the storage directory."); } } + if (config_.gc.type == Config::Gc::Type::PERIODIC) { - gc_runner_.Run("Storage GC", config_.gc.interval, [this] { this->CollectGarbage(); }); + // TODO: move out of storage have one global gc_runner_ + gc_runner_.Run("Storage GC", config_.gc.interval, [this] { + this->FreeMemory(std::unique_lock{main_lock_, std::defer_lock}); + }); + gc_jemalloc_runner_.Run("Jemalloc GC", config_.gc.interval, [] { memory::PurgeUnusedMemory(); }); } if (timestamp_ == kTimestampInitialId) { commit_log_.emplace(); @@ -116,6 +122,7 @@ InMemoryStorage::InMemoryStorage(Config config) : InMemoryStorage(config, Storag InMemoryStorage::~InMemoryStorage() { if (config_.gc.type == Config::Gc::Type::PERIODIC) { gc_runner_.Stop(); + gc_jemalloc_runner_.Stop(); } { // Stop replication (Stop all clients or stop the REPLICA server) @@ -1210,7 +1217,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_ main_lock_.lock_shared(); } } else { - MG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_"); + DMG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_"); } utils::OnScopeExit lock_releaser{[&] { diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 1d16eadf1..bfb445332 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -418,6 +418,7 @@ class InMemoryStorage final : public Storage { std::optional commit_log_; utils::Scheduler gc_runner_; + utils::Scheduler gc_jemalloc_runner_; std::mutex gc_lock_; using BondPmrLd = Bond>; diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 71d80b7ed..28fe94559 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -71,6 +71,7 @@ add_subdirectory(query_modules) add_subdirectory(constraints) add_subdirectory(inspect_query) add_subdirectory(queries) +add_subdirectory(garbage_collection) copy_e2e_python_files(pytest_runner pytest_runner.sh "") copy_e2e_python_files(x x.sh "") diff --git a/tests/e2e/garbage_collection/CMakeLists.txt b/tests/e2e/garbage_collection/CMakeLists.txt new file mode 100644 index 000000000..690edf344 --- /dev/null +++ b/tests/e2e/garbage_collection/CMakeLists.txt @@ -0,0 +1,7 @@ +function(garbage_collection_e2e_python_files FILE_NAME) + copy_e2e_python_files(garbage_collection ${FILE_NAME}) +endfunction() + +garbage_collection_e2e_python_files(common.py) +garbage_collection_e2e_python_files(conftest.py) +garbage_collection_e2e_python_files(gc_periodic.py) diff --git a/tests/e2e/garbage_collection/common.py b/tests/e2e/garbage_collection/common.py new file mode 100644 index 000000000..dedf18dc3 --- /dev/null +++ b/tests/e2e/garbage_collection/common.py @@ -0,0 +1,25 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import typing + +import mgclient + + +def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]: + cursor.execute(query, params) + return cursor.fetchall() + + +def connect(**kwargs) -> mgclient.Connection: + connection = mgclient.connect(host="localhost", port=7687, **kwargs) + connection.autocommit = True + return connection diff --git a/tests/e2e/garbage_collection/conftest.py b/tests/e2e/garbage_collection/conftest.py new file mode 100644 index 000000000..a4ec62c9f --- /dev/null +++ b/tests/e2e/garbage_collection/conftest.py @@ -0,0 +1,21 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import pytest +from common import connect, execute_and_fetch_all + + +@pytest.fixture(autouse=True) +def connection(): + connection = connect() + yield connection + cursor = connection.cursor() + execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n") diff --git a/tests/e2e/garbage_collection/gc_periodic.py b/tests/e2e/garbage_collection/gc_periodic.py new file mode 100644 index 000000000..6903960be --- /dev/null +++ b/tests/e2e/garbage_collection/gc_periodic.py @@ -0,0 +1,59 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import re +import sys +import time + +import pytest +from common import execute_and_fetch_all + + +def remove_non_numeric_suffix(text): + match = re.search(r"\D*$", text) + if match: + non_numeric_suffix = match.group(0) + return text[: -len(non_numeric_suffix)] + else: + return text + + +def get_memory_from_list(list): + for list_item in list: + if list_item[0] == "memory_tracked": + return float(remove_non_numeric_suffix(list_item[1])) + return None + + +def get_memory(cursor): + return get_memory_from_list(execute_and_fetch_all(cursor, "SHOW STORAGE INFO")) + + +def test_gc_periodic(connection): + """ + This test checks that periodic gc works. + It does so by checking that the allocated memory is lowered by at least 1/4 of the memory allocated by creating nodes. + If we choose a number a high number the test will become flaky because the memory only gets fully cleared after a while + due to jemalloc holding some memory for a while. If we'd wait for jemalloc to fully release the memory the test would take too long. + """ + cursor = connection.cursor() + + memory_pre_creation = get_memory(cursor) + execute_and_fetch_all(cursor, "UNWIND range(1, 1000) AS index CREATE (:Node);") + memory_after_creation = get_memory(cursor) + time.sleep(2) + memory_after_gc = get_memory(cursor) + + assert memory_after_gc < memory_pre_creation + (memory_after_creation - memory_pre_creation) / 4 * 3 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/garbage_collection/workloads.yaml b/tests/e2e/garbage_collection/workloads.yaml new file mode 100644 index 000000000..395ba83d9 --- /dev/null +++ b/tests/e2e/garbage_collection/workloads.yaml @@ -0,0 +1,19 @@ +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--storage-gc-cycle-sec=2" + +in_memory_cluster: &in_memory_cluster + cluster: + main: + args: *args + log_file: "garbage_collection-e2e.log" + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Garbage collection" + binary: "tests/e2e/pytest_runner.sh" + args: ["garbage_collection/gc_periodic.py"] + <<: *in_memory_cluster From c037cddb0ec57ca0cdebfd7bd0c1af024ea66164 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Tue, 14 Nov 2023 23:23:06 +0100 Subject: [PATCH 16/52] Add granular index and constraint recovery info (#1480) --- src/dbms/inmemory/replication_handlers.cpp | 3 +- src/storage/v2/durability/durability.cpp | 34 +++++++++++++++------- src/storage/v2/durability/durability.hpp | 2 +- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/dbms/inmemory/replication_handlers.cpp b/src/dbms/inmemory/replication_handlers.cpp index b50163c55..ce1f6da20 100644 --- a/src/dbms/inmemory/replication_handlers.cpp +++ b/src/dbms/inmemory/replication_handlers.cpp @@ -220,7 +220,8 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle spdlog::trace("Recovering indices and constraints from snapshot."); storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_, - &storage->constraints_, &storage->vertices_); + &storage->constraints_, &storage->vertices_, + storage->name_id_mapper_.get()); } catch (const storage::durability::RecoveryFailure &e) { LOG_FATAL("Couldn't load the snapshot because of: {}", e.what()); } diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index decbfd14a..1240bd52e 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -130,16 +130,17 @@ std::optional> GetWalFiles(const std::filesystem: // recovery process. void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints, utils::SkipList *vertices, + NameIdMapper *name_id_mapper, const std::optional ¶llel_exec_info) { spdlog::info("Recreating indices from metadata."); // Recover label indices. spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size()); auto *mem_label_index = static_cast(indices->label_index_.get()); for (const auto &item : indices_constraints.indices.label) { - if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) + if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) { throw RecoveryFailure("The label index must be created here!"); - - spdlog::info("A label index is recreated from metadata."); + } + spdlog::info("Index on :{} is recreated from metadata", name_id_mapper->IdToName(item.AsUint())); } spdlog::info("Label indices are recreated."); @@ -148,7 +149,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Recreating {} label index statistics from metadata.", indices_constraints.indices.label_stats.size()); for (const auto &item : indices_constraints.indices.label_stats) { mem_label_index->SetIndexStats(item.first, item.second); - spdlog::info("A label index statistics is recreated from metadata."); + spdlog::info("Statistics for index on :{} are recreated from metadata", + name_id_mapper->IdToName(item.first.AsUint())); } spdlog::info("Label indices statistics are recreated."); @@ -159,7 +161,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ for (const auto &item : indices_constraints.indices.label_property) { if (!mem_label_property_index->CreateIndex(item.first, item.second, vertices->access(), parallel_exec_info)) throw RecoveryFailure("The label+property index must be created here!"); - spdlog::info("A label+property index is recreated from metadata."); + spdlog::info("Index on :{}({}) is recreated from metadata", name_id_mapper->IdToName(item.first.AsUint()), + name_id_mapper->IdToName(item.second.AsUint())); } spdlog::info("Label+property indices are recreated."); @@ -171,7 +174,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ const auto property_id = item.second.first; const auto &stats = item.second.second; mem_label_property_index->SetIndexStats({label_id, property_id}, stats); - spdlog::info("A label+property index statistics is recreated from metadata."); + spdlog::info("Statistics for index on :{}({}) are recreated from metadata", + name_id_mapper->IdToName(label_id.AsUint()), name_id_mapper->IdToName(property_id.AsUint())); } spdlog::info("Label+property indices statistics are recreated."); @@ -191,8 +195,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ } constraints->existence_constraints_->InsertConstraint(label, property); - - spdlog::info("A existence constraint is recreated from metadata."); + spdlog::info("Existence constraint on :{}({}) is recreated from metadata", name_id_mapper->IdToName(label.AsUint()), + name_id_mapper->IdToName(property.AsUint())); } spdlog::info("Existence constraints are recreated from metadata."); @@ -203,7 +207,15 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ auto ret = mem_unique_constraints->CreateConstraint(item.first, item.second, vertices->access()); if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) throw RecoveryFailure("The unique constraint must be created here!"); - spdlog::info("A unique constraint is recreated from metadata."); + + std::vector property_names; + property_names.reserve(item.second.size()); + for (const auto &prop : item.second) { + property_names.emplace_back(name_id_mapper->IdToName(prop.AsUint())); + } + const auto property_names_joined = utils::Join(property_names, ","); + spdlog::info("Unique constraint on :{}({}) is recreated from metadata", + name_id_mapper->IdToName(item.first.AsUint()), property_names_joined); } spdlog::info("Unique constraints are recreated from metadata."); spdlog::info("Constraints are recreated from metadata."); @@ -270,7 +282,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) : std::nullopt; - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info); + RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); return recovered_snapshot->recovery_info; } } else { @@ -402,7 +414,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) : std::nullopt; - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info); + RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us, std::chrono::duration_cast(timer.Elapsed()).count()); diff --git a/src/storage/v2/durability/durability.hpp b/src/storage/v2/durability/durability.hpp index 8b735f02a..8bb1223c4 100644 --- a/src/storage/v2/durability/durability.hpp +++ b/src/storage/v2/durability/durability.hpp @@ -104,7 +104,7 @@ using ParallelizedIndexCreationInfo = /// @throw RecoveryFailure void RecoverIndicesAndConstraints( const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints, - utils::SkipList *vertices, + utils::SkipList *vertices, NameIdMapper *name_id_mapper, const std::optional ¶llel_exec_info = std::nullopt); /// Recovers data either from a snapshot and/or WAL files. From d3f4c3536299ea2da2e39123b087500553150411 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:42:04 +0100 Subject: [PATCH 17/52] Add OOM enabler for MG procedure (#1401) --- src/query/plan/operator.cpp | 2 + src/query/plan/rule_based_planner.hpp | 9 +- src/query/procedure/mg_procedure_impl.cpp | 3 + tests/e2e/memory/CMakeLists.txt | 9 +- ..._limit_global_multi_thread_proc_create.cpp | 67 +++++++++++++ .../memory_limit_global_thread_alloc_proc.cpp | 68 +++++++++++++ .../procedure_memory_limit_multi_proc.cpp | 7 +- tests/e2e/memory/procedures/CMakeLists.txt | 9 +- ..._memory_limit_multi_thread_create_proc.cpp | 95 +++++++++++++++++++ .../global_memory_limit_thread_proc.cpp | 92 ++++++++++++++++++ .../memory/procedures/proc_memory_limit.cpp | 3 + .../query_memory_limit_proc_multi_thread.cpp | 3 +- tests/e2e/memory/workloads.yaml | 15 ++- 13 files changed, 368 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp create mode 100644 tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp create mode 100644 tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp create mode 100644 tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 238638737..52e15e928 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -57,6 +57,7 @@ #include "utils/likely.hpp" #include "utils/logging.hpp" #include "utils/memory.hpp" +#include "utils/memory_tracker.hpp" #include "utils/message.hpp" #include "utils/on_scope_exit.hpp" #include "utils/pmr/deque.hpp" @@ -4859,6 +4860,7 @@ class CallProcedureCursor : public Cursor { result_signature_size_ = result_->signature->size(); result_->signature = nullptr; if (result_->error_msg) { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker; throw QueryRuntimeException("{}: {}", self_->procedure_name_, *result_->error_msg); } result_row_it_ = result_->rows.begin(); diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index afa060b9d..bdac76a93 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -176,7 +176,10 @@ class RuleBasedPlanner { PlanResult Plan(const QueryParts &query_parts) { auto &context = *context_; std::unique_ptr final_plan; - + // procedures need to start from 1 + // due to swapping mechanism of procedure + // tracking + uint64_t procedure_id = 1; for (const auto &query_part : query_parts.query_parts) { std::unique_ptr input_op; @@ -186,10 +189,6 @@ class RuleBasedPlanner { uint64_t merge_id = 0; uint64_t subquery_id = 0; - // procedures need to start from 1 - // due to swapping mechanism of procedure - // tracking - uint64_t procedure_id = 1; for (const auto &clause : single_query_part.remaining_clauses) { MG_ASSERT(!utils::IsSubtype(*clause, Match::kType), "Unexpected Match in remaining clauses"); diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index 2a657aeb3..2a176f2ed 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -38,6 +38,7 @@ #include "utils/logging.hpp" #include "utils/math.hpp" #include "utils/memory.hpp" +#include "utils/memory_tracker.hpp" #include "utils/string.hpp" #include "utils/temporal.hpp" #include "utils/variant_helpers.hpp" @@ -158,6 +159,7 @@ template [[nodiscard]] mgp_error WrapExceptions(TFunc &&func, Args &&...args) noexcept { static_assert(sizeof...(args) <= 1, "WrapExceptions should have only one or zero parameter!"); try { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_enabler; WrapExceptionsHelper(std::forward(func), std::forward(args)...); } catch (const DeletedObjectException &neoe) { spdlog::error("Deleted object error during mg API call: {}", neoe.what()); @@ -1544,6 +1546,7 @@ mgp_error mgp_duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory mgp_error mgp_result_set_error_msg(mgp_result *res, const char *msg) { return WrapExceptions([=] { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker{}; auto *memory = res->rows.get_allocator().GetMemoryResource(); res->error_msg.emplace(msg, memory); }); diff --git a/tests/e2e/memory/CMakeLists.txt b/tests/e2e/memory/CMakeLists.txt index 327f09106..3c4cdc279 100644 --- a/tests/e2e/memory/CMakeLists.txt +++ b/tests/e2e/memory/CMakeLists.txt @@ -2,6 +2,8 @@ add_subdirectory(procedures) find_package(gflags REQUIRED) +# Global memory limit + add_executable(memgraph__e2e__memory__control memory_control.cpp) target_link_libraries(memgraph__e2e__memory__control gflags mgclient mg-utils mg-io Threads::Threads) @@ -20,6 +22,12 @@ target_link_libraries(memgraph__e2e__memory__limit_accumulation gflags mgclient add_executable(memgraph__e2e__memory__limit_edge_create memory_limit_edge_create.cpp) target_link_libraries(memgraph__e2e__memory__limit_edge_create gflags mgclient mg-utils mg-io) +add_executable(memgraph__e2e__memory_limit_global_multi_thread_proc_create memory_limit_global_multi_thread_proc_create.cpp) +target_link_libraries(memgraph__e2e__memory_limit_global_multi_thread_proc_create gflags mgclient mg-utils mg-io) + +add_executable(memgraph__e2e__memory_limit_global_thread_alloc_proc memory_limit_global_thread_alloc_proc.cpp) +target_link_libraries(memgraph__e2e__memory_limit_global_thread_alloc_proc gflags mgclient mg-utils mg-io) + # Query memory limit tests add_executable(memgraph__e2e__memory__limit_query_alloc_proc_multi_thread query_memory_limit_proc_multi_thread.cpp) @@ -34,7 +42,6 @@ target_link_libraries(memgraph__e2e__memory__limit_query_alloc_proc gflags mgcli add_executable(memgraph__e2e__memory__limit_query_alloc_create_multi_thread query_memory_limit_multi_thread.cpp) target_link_libraries(memgraph__e2e__memory__limit_query_alloc_create_multi_thread gflags mgclient mg-utils mg-io Threads::Threads) - # Procedure memory limit tests add_executable(memgraph__e2e__procedure_memory_limit procedure_memory_limit.cpp) diff --git a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp new file mode 100644 index 000000000..2132fbb16 --- /dev/null +++ b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp @@ -0,0 +1,67 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include "utils/logging.hpp" +#include "utils/timer.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); +DEFINE_uint64(timeout, 120, "Timeout seconds"); +DEFINE_bool(multi_db, false, "Run test in multi db environment"); + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread Create For Local Allocators"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + mg::Client::Init(); + + auto client = + mg::Client::Connect({.host = "127.0.0.1", .port = static_cast(FLAGS_bolt_port), .use_ssl = false}); + if (!client) { + LOG_FATAL("Failed to connect!"); + } + + if (FLAGS_multi_db) { + client->Execute("CREATE DATABASE clean;"); + client->DiscardAll(); + client->Execute("USE DATABASE clean;"); + client->DiscardAll(); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + } + + bool error{false}; + try { + client->Execute( + "CALL libglobal_memory_limit_multi_thread_create_proc.multi_create() PROCEDURE MEMORY UNLIMITED YIELD " + "allocated_all RETURN allocated_all " + "QUERY MEMORY LIMIT 50MB;"); + auto result_rows = client->FetchAll(); + if (result_rows) { + auto row = *result_rows->begin(); + error = row[0].ValueBool() == false; + } + + } catch (const std::exception &e) { + error = true; + } + + MG_ASSERT(error, "Error should have happend"); + + return 0; +} diff --git a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp new file mode 100644 index 000000000..0c2eb1ee6 --- /dev/null +++ b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp @@ -0,0 +1,68 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include "utils/logging.hpp" +#include "utils/timer.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); +DEFINE_uint64(timeout, 120, "Timeout seconds"); +DEFINE_bool(multi_db, false, "Run test in multi db environment"); + +// Test checks path of throwing error from different thread +// than main thread which started test +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread For Procedures For Local Allocators"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + mg::Client::Init(); + + auto client = + mg::Client::Connect({.host = "127.0.0.1", .port = static_cast(FLAGS_bolt_port), .use_ssl = false}); + if (!client) { + LOG_FATAL("Failed to connect!"); + } + + if (FLAGS_multi_db) { + client->Execute("CREATE DATABASE clean;"); + client->DiscardAll(); + client->Execute("USE DATABASE clean;"); + client->DiscardAll(); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + } + + bool error{false}; + try { + client->Execute( + "CALL libglobal_memory_limit_thread_proc.thread() YIELD allocated_all RETURN allocated_all QUERY MEMORY LIMIT " + "100MB;"); + auto result_rows = client->FetchAll(); + if (result_rows) { + auto row = *result_rows->begin(); + error = row[0].ValueBool() == false; + } + + } catch (const std::exception &e) { + error = true; + } + + MG_ASSERT(error, "Error should have happend"); + + return 0; +} diff --git a/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp b/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp index 118ff41ce..f850ec6c2 100644 --- a/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp +++ b/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "utils/logging.hpp" @@ -53,11 +54,9 @@ int main(int argc, char **argv) { "CALL libproc_memory_limit.alloc_32_mib() PROCEDURE MEMORY LIMIT 10MB YIELD allocated AS allocated_2 RETURN " "allocated_1, allocated_2"); auto result_rows = client->FetchAll(); - if (result_rows) { - auto row = *result_rows->begin(); - test_passed = row[0].ValueBool() == true && row[0].ValueBool() == false; + if (result_rows && result_rows->empty()) { + test_passed = true; } - } catch (const std::exception &e) { test_passed = true; } diff --git a/tests/e2e/memory/procedures/CMakeLists.txt b/tests/e2e/memory/procedures/CMakeLists.txt index 84c56f414..df7acee31 100644 --- a/tests/e2e/memory/procedures/CMakeLists.txt +++ b/tests/e2e/memory/procedures/CMakeLists.txt @@ -4,16 +4,21 @@ target_include_directories(global_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/inclu add_library(global_memory_limit_proc SHARED global_memory_limit_proc.c) target_include_directories(global_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) - add_library(query_memory_limit_proc_multi_thread SHARED query_memory_limit_proc_multi_thread.cpp) target_include_directories(query_memory_limit_proc_multi_thread PRIVATE ${CMAKE_SOURCE_DIR}/include) target_link_libraries(query_memory_limit_proc_multi_thread mg-utils) - add_library(query_memory_limit_proc SHARED query_memory_limit_proc.cpp) target_include_directories(query_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) target_link_libraries(query_memory_limit_proc mg-utils) +add_library(global_memory_limit_thread_proc SHARED global_memory_limit_thread_proc.cpp) +target_include_directories(global_memory_limit_thread_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(global_memory_limit_thread_proc mg-utils) + +add_library(global_memory_limit_multi_thread_create_proc SHARED global_memory_limit_multi_thread_create_proc.cpp) +target_include_directories(global_memory_limit_multi_thread_create_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(global_memory_limit_multi_thread_create_proc mg-utils) add_library(proc_memory_limit SHARED proc_memory_limit.cpp) target_include_directories(proc_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/include) diff --git a/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp b/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp new file mode 100644 index 000000000..2ccaac631 --- /dev/null +++ b/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp @@ -0,0 +1,95 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mg_procedure.h" +#include "mgp.hpp" +#include "utils/on_scope_exit.hpp" + +// change communication between threads with feature and promise +std::atomic created_vertices{0}; +constexpr int num_vertices_per_thread{100'000}; +constexpr int num_threads{2}; + +void CallCreate(mgp_graph *graph, mgp_memory *memory) { + [[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph); + for (int i = 0; i < num_vertices_per_thread; i++) { + struct mgp_vertex *vertex{nullptr}; + auto enum_error = mgp_graph_create_vertex(graph, memory, &vertex); + if (enum_error != mgp_error::MGP_ERROR_NO_ERROR) { + break; + } + created_vertices.fetch_add(1, std::memory_order_acq_rel); + } + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); +} + +void AllocFunc(mgp_graph *graph, mgp_memory *memory) { + try { + CallCreate(graph, memory); + } catch (const std::exception &e) { + return; + } +} + +void MultiCreate(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto arguments = mgp::List(args); + const auto record_factory = mgp::RecordFactory(result); + try { + std::vector threads; + + for (int i = 0; i < 2; i++) { + threads.emplace_back(AllocFunc, memgraph_graph, memory); + } + + for (int i = 0; i < num_threads; i++) { + threads[i].join(); + } + if (created_vertices.load(std::memory_order_acquire) != num_vertices_per_thread * num_threads) { + record_factory.SetErrorMessage("Unable to allocate"); + return; + } + + auto new_record = record_factory.NewRecord(); + new_record.Insert("allocated_all", + created_vertices.load(std::memory_order_acquire) == num_vertices_per_thread * num_threads); + } catch (std::exception &e) { + record_factory.SetErrorMessage(e.what()); + } +} + +extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { + try { + mgp::MemoryDispatcherGuard guard{memory}; + + AddProcedure(MultiCreate, std::string("multi_create").c_str(), mgp::ProcedureType::Write, {}, + {mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory); + + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp b/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp new file mode 100644 index 000000000..95bf19f9d --- /dev/null +++ b/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp @@ -0,0 +1,92 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mg_procedure.h" +#include "mgp.hpp" +#include "utils/on_scope_exit.hpp" + +enum mgp_error Alloc(mgp_memory *memory, void *ptr) { + const size_t mb_size_512 = 1 << 29; + + return mgp_alloc(memory, mb_size_512, (void **)(&ptr)); +} + +// change communication between threads with feature and promise +std::atomic num_allocations{0}; +void *ptr_; + +void AllocFunc(mgp_memory *memory, mgp_graph *graph) { + try { + [[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph); + enum mgp_error alloc_err { mgp_error::MGP_ERROR_NO_ERROR }; + alloc_err = Alloc(memory, ptr_); + if (alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { + num_allocations.fetch_add(1, std::memory_order_relaxed); + } + if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) { + assert(false); + } + } catch (const std::exception &e) { + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); + assert(false); + } + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); +} + +void Thread(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto arguments = mgp::List(args); + const auto record_factory = mgp::RecordFactory(result); + num_allocations.store(0, std::memory_order_relaxed); + try { + std::thread thread{AllocFunc, memory, memgraph_graph}; + + thread.join(); + + if (ptr_ != nullptr) { + mgp_free(memory, ptr_); + } + + auto new_record = record_factory.NewRecord(); + + new_record.Insert("allocated_all", num_allocations.load(std::memory_order_relaxed) == 1); + } catch (std::exception &e) { + record_factory.SetErrorMessage(e.what()); + } +} + +extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { + try { + mgp::memory = memory; + + AddProcedure(Thread, std::string("thread").c_str(), mgp::ProcedureType::Read, {}, + {mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory); + + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/memory/procedures/proc_memory_limit.cpp b/tests/e2e/memory/procedures/proc_memory_limit.cpp index 9f36dfaa4..d78407222 100644 --- a/tests/e2e/memory/procedures/proc_memory_limit.cpp +++ b/tests/e2e/memory/procedures/proc_memory_limit.cpp @@ -75,6 +75,9 @@ void Alloc_32_MiB(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, }}; const enum mgp_error alloc_err = Alloc_32(memory, ptr); + if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) { + record_factory.SetErrorMessage("Unable to allocate"); + } auto new_record = record_factory.NewRecord(); new_record.Insert("allocated", alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE); } catch (std::exception &e) { diff --git a/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp b/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp index ffc509ff3..0a1d8f125 100644 --- a/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp +++ b/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp @@ -13,7 +13,7 @@ #include #include #include -#include + #include #include #include @@ -22,6 +22,7 @@ #include #include "mg_procedure.h" +#include "mgp.hpp" #include "utils/on_scope_exit.hpp" enum mgp_error Alloc(void *ptr) { diff --git a/tests/e2e/memory/workloads.yaml b/tests/e2e/memory/workloads.yaml index e84faccf0..d826175ed 100644 --- a/tests/e2e/memory/workloads.yaml +++ b/tests/e2e/memory/workloads.yaml @@ -144,6 +144,7 @@ workloads: binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create_multi_thread" args: ["--bolt-port", *bolt_port] <<: *in_memory_query_limit_cluster + - name: "Memory control for detach delete" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_delete" args: ["--bolt-port", *bolt_port] @@ -174,6 +175,18 @@ workloads: args: ["--bolt-port", *bolt_port] <<: *disk_450_MiB_limit_cluster + - name: "Memory control for create from multi thread proc create" + binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_multi_thread_proc_create" + proc: "tests/e2e/memory/procedures/" + args: ["--bolt-port", *bolt_port] + <<: *in_memory_cluster + + - name: "Memory control for memory limit global thread alloc" + binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_thread_alloc_proc" + proc: "tests/e2e/memory/procedures/" + args: ["--bolt-port", *bolt_port] + <<: *in_memory_cluster + - name: "Procedure memory control for single procedure" binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit" proc: "tests/e2e/memory/procedures/" @@ -181,7 +194,7 @@ workloads: <<: *in_memory_limited_global_limit_cluster - name: "Procedure memory control for multiple procedures" - binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit" + binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit_multi_proc" proc: "tests/e2e/memory/procedures/" args: ["--bolt-port", *bolt_port] <<: *in_memory_limited_global_limit_cluster From 645568a75bc55411c61f1499d6a810c79bebba44 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:01:44 +0100 Subject: [PATCH 18/52] Remove default memory limit on procedures (#1506) * remove default limit on procedures * fix bug on GraphQL also --- .../frontend/ast/cypher_main_visitor.cpp | 10 ++++---- tests/unit/cypher_main_visitor.cpp | 23 ------------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 6b75061fb..4bf7f36fd 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -1216,10 +1216,6 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur call_proc->memory_limit_ = memory_limit_info->first; call_proc->memory_scale_ = memory_limit_info->second; } - } else { - // Default to 100 MB - call_proc->memory_limit_ = storage_->Create(TypedValue(100)); - call_proc->memory_scale_ = 1024U * 1024U; } const auto &maybe_found = @@ -1240,11 +1236,13 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); } } - call_proc->is_write_ = maybe_found->second->info.is_write; + if (maybe_found) { + call_proc->is_write_ = maybe_found->second->info.is_write; + } auto *yield_ctx = ctx->yieldProcedureResults(); if (!yield_ctx) { - if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) { + if ((maybe_found && !maybe_found->second->results.empty()) && !call_proc->void_procedure_) { throw SemanticException( "CALL without YIELD may only be used on procedures which do not " "return any result fields."); diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index acbd1fe41..1d7f236a2 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -2833,18 +2833,6 @@ TEST_P(CypherMainVisitorTest, DumpDatabase) { ASSERT_TRUE(query); } -namespace { -template -void CheckCallProcedureDefaultMemoryLimit(const TAst &ast, const CallProcedure &call_proc) { - // Should be 100 MB - auto *literal = dynamic_cast(call_proc.memory_limit_); - ASSERT_TRUE(literal); - TypedValue value(literal->value_); - ASSERT_TRUE(TypedValue::BoolEqual{}(value, TypedValue(100))); - ASSERT_EQ(call_proc.memory_scale_, 1024 * 1024); -} -} // namespace - TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) { AddProc(*mock_module_with_dots_in_name, "proc", {}, {"res"}, ProcedureType::WRITE); auto &ast_generator = *GetParam(); @@ -2868,7 +2856,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) { @@ -2894,7 +2881,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) { @@ -2926,7 +2912,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) { std::vector expected_names{"fst", "field-with-dashes", "last_field"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); }; check_proc(ProcedureType::READ); check_proc(ProcedureType::WRITE); @@ -2959,7 +2944,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldAliasedFields) { ASSERT_EQ(identifier_names, aliased_names); std::vector field_names{"fst", "snd", "thrd"}; ASSERT_EQ(call_proc->result_fields_, field_names); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) { @@ -2986,7 +2970,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) { @@ -3008,7 +2991,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) { } ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable")); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) { @@ -3033,7 +3015,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) { } ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable")); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) { @@ -3049,7 +3030,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) { ASSERT_TRUE(call_proc->arguments_.empty()); ASSERT_TRUE(call_proc->result_fields_.empty()); ASSERT_TRUE(call_proc->result_identifiers_.empty()); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithMemoryLimitWithoutYield) { @@ -3183,7 +3163,6 @@ void CheckParsedCallProcedure(const CypherQuery &query, Base &ast_generator, EXPECT_EQ(identifier_names, args_as_str); EXPECT_EQ(identifier_names, call_proc->result_fields_); ASSERT_EQ(call_proc->is_write_, type == ProcedureType::WRITE); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); }; } // namespace @@ -3577,7 +3556,6 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) { auto *single_query = query->single_query_; ASSERT_EQ(single_query->clauses_.size(), 2U); auto *call_proc = dynamic_cast(single_query->clauses_[0]); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } { @@ -3638,7 +3616,6 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) { auto *single_query = query->single_query_; ASSERT_EQ(single_query->clauses_.size(), 1U); auto *call_proc = dynamic_cast(single_query->clauses_[0]); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } } From 6053a91ef86bfa3fb321f99d7c3d5ba2bce5db69 Mon Sep 17 00:00:00 2001 From: imilinovic <44698587+imilinovic@users.noreply.github.com> Date: Fri, 17 Nov 2023 23:06:46 +0100 Subject: [PATCH 19/52] Fix flaky GC test (#1521) --- tests/e2e/garbage_collection/gc_periodic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/garbage_collection/gc_periodic.py b/tests/e2e/garbage_collection/gc_periodic.py index 6903960be..a93846a68 100644 --- a/tests/e2e/garbage_collection/gc_periodic.py +++ b/tests/e2e/garbage_collection/gc_periodic.py @@ -49,7 +49,7 @@ def test_gc_periodic(connection): memory_pre_creation = get_memory(cursor) execute_and_fetch_all(cursor, "UNWIND range(1, 1000) AS index CREATE (:Node);") memory_after_creation = get_memory(cursor) - time.sleep(2) + time.sleep(5) memory_after_gc = get_memory(cursor) assert memory_after_gc < memory_pre_creation + (memory_after_creation - memory_pre_creation) / 4 * 3 From c31a7f96482fb1eedabc1d4d97d4a20b3551a414 Mon Sep 17 00:00:00 2001 From: Hal Eisen Date: Fri, 17 Nov 2023 17:25:11 -0800 Subject: [PATCH 20/52] First draft of a sonarcloud properties file --- .sonarcloud.properties | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .sonarcloud.properties diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..09e34ed00 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,22 @@ +# Path to sources +sonar.sources = src/ +# sonar.exclusions= +# sonar.inclusions= + +# Path to tests +sonar.tests = tests/ +# sonar.test.exclusions= +# sonar.test.inclusions= + +# Source encoding +# sonar.sourceEncoding= + +# Exclusions for copy-paste detection +# sonar.cpd.exclusions= + +# Python version (for python projects only) +# sonar.python.version= + +# C++ standard version (for C++ projects only) +# If not specified, it defaults to the latest supported standard +# sonar.cfamily.reportingCppStandardOverride=c++98|c++11|c++14|c++17|c++20 From d03fafcef6fae57c7e456372f64ea227fb84b316 Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 20 Nov 2023 11:52:17 +0100 Subject: [PATCH 21/52] Aggregations return empty result when used with group by (#1531) --- src/query/plan/operator.cpp | 8 +---- src/query/plan/operator.hpp | 2 -- .../memgraph_V1/features/aggregations.feature | 4 +-- .../features/aggregations.feature | 4 +-- .../unit/query_plan_accumulate_aggregate.cpp | 34 ++++++------------- 5 files changed, 13 insertions(+), 39 deletions(-) diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 52e15e928..b68810ad7 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -3464,7 +3464,7 @@ class AggregateCursor : public Cursor { SCOPED_PROFILE_OP_BY_REF(self_); if (!pulled_all_input_) { - if (!ProcessAll(&frame, &context) && self_.AreAllAggregationsForCollecting()) return false; + if (!ProcessAll(&frame, &context) && !self_.group_by_.empty()) return false; pulled_all_input_ = true; aggregation_it_ = aggregation_.begin(); @@ -3824,12 +3824,6 @@ UniqueCursorPtr Aggregate::MakeCursor(utils::MemoryResource *mem) const { return MakeUniqueCursorPtr(mem, *this, mem); } -auto Aggregate::AreAllAggregationsForCollecting() const -> bool { - return std::all_of(aggregations_.begin(), aggregations_.end(), [](const auto &agg) { - return agg.op == Aggregation::Op::COLLECT_LIST || agg.op == Aggregation::Op::COLLECT_MAP; - }); -} - Skip::Skip(const std::shared_ptr &input, Expression *expression) : input_(input), expression_(expression) {} diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index ba844796a..7bb971752 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -1759,8 +1759,6 @@ class Aggregate : public memgraph::query::plan::LogicalOperator { Aggregate(const std::shared_ptr &input, const std::vector &aggregations, const std::vector &group_by, const std::vector &remember); - auto AreAllAggregationsForCollecting() const -> bool; - bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; diff --git a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature index cff138432..80b0ca69a 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature @@ -425,6 +425,4 @@ Feature: Aggregations """ MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id """ - Then the result should be: - | id | - | null | + Then the result should be empty diff --git a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature index cff138432..80b0ca69a 100644 --- a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature @@ -425,6 +425,4 @@ Feature: Aggregations """ MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id """ - Then the result should be: - | id | - | null | + Then the result should be empty diff --git a/tests/unit/query_plan_accumulate_aggregate.cpp b/tests/unit/query_plan_accumulate_aggregate.cpp index bbf3e0311..b1e9a62d0 100644 --- a/tests/unit/query_plan_accumulate_aggregate.cpp +++ b/tests/unit/query_plan_accumulate_aggregate.cpp @@ -250,30 +250,23 @@ TYPED_TEST(QueryPlanAggregateOps, WithData) { TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithGroupBy) { { auto results = this->AggregationResults(true, false, {Aggregation::Op::COUNT}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::SUM}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::AVG}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::MIN}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::MAX}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::COLLECT_LIST}); @@ -666,30 +659,23 @@ TYPED_TEST(QueryPlanAggregateOps, WithDataDistinct) { TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithDistinctAndWithGroupBy) { { auto results = this->AggregationResults(true, true, {Aggregation::Op::COUNT}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::SUM}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::AVG}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::MIN}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::MAX}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::COLLECT_LIST}); From eff857447acac88821cb627d0aff36ca5774cdc3 Mon Sep 17 00:00:00 2001 From: Hal Eisen Date: Mon, 20 Nov 2023 08:16:35 -0800 Subject: [PATCH 22/52] Refine scope to pull in 'include' and 'query_module' directories, in addition to src --- .sonarcloud.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.sonarcloud.properties b/.sonarcloud.properties index 09e34ed00..2c53f52f2 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -1,7 +1,7 @@ # Path to sources -sonar.sources = src/ +sonar.sources = . # sonar.exclusions= -# sonar.inclusions= +sonar.inclusions=src,include,query_modules # Path to tests sonar.tests = tests/ From 1d90b60f56b8acb1ac5a8003da362bd75caa1ed4 Mon Sep 17 00:00:00 2001 From: Andi Date: Tue, 21 Nov 2023 09:19:50 +0100 Subject: [PATCH 23/52] Add schema.assert (#1485) --- include/_mgp.hpp | 48 +++ include/mg_procedure.h | 59 +++ include/mgp.hpp | 82 +++- query_modules/schema.cpp | 465 +++++++++++++++++++++- src/query/db_accessor.cpp | 2 + src/query/db_accessor.hpp | 2 + src/query/interpreter.cpp | 5 +- src/query/procedure/mg_procedure_impl.cpp | 327 +++++++++++++++ src/storage/v2/inmemory/storage.cpp | 16 +- tests/e2e/query_modules/schema_test.py | 404 +++++++++++++++++++ tests/e2e/query_modules/workloads.yaml | 2 +- 11 files changed, 1373 insertions(+), 39 deletions(-) diff --git a/include/_mgp.hpp b/include/_mgp.hpp index fd286b6c6..58685b440 100644 --- a/include/_mgp.hpp +++ b/include/_mgp.hpp @@ -236,6 +236,54 @@ inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke(mgp // mgp_graph +inline bool create_label_index(mgp_graph *graph, const char *label) { + return MgInvoke(mgp_create_label_index, graph, label); +} + +inline bool drop_label_index(mgp_graph *graph, const char *label) { + return MgInvoke(mgp_drop_label_index, graph, label); +} + +inline mgp_list *list_all_label_indices(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_label_indices, graph, memory); +} + +inline bool create_label_property_index(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_create_label_property_index, graph, label, property); +} + +inline bool drop_label_property_index(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_drop_label_property_index, graph, label, property); +} + +inline mgp_list *list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_label_property_indices, graph, memory); +} + +inline bool create_existence_constraint(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_create_existence_constraint, graph, label, property); +} + +inline bool drop_existence_constraint(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_drop_existence_constraint, graph, label, property); +} + +inline mgp_list *list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_existence_constraints, graph, memory); +} + +inline bool create_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) { + return MgInvoke(mgp_create_unique_constraint, memgraph_graph, label, properties); +} + +inline bool drop_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) { + return MgInvoke(mgp_drop_unique_constraint, memgraph_graph, label, properties); +} + +inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_unique_constraints, graph, memory); +} + inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke(mgp_graph_is_mutable, graph); } inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) { diff --git a/include/mg_procedure.h b/include/mg_procedure.h index 0bd831174..857c5f4dd 100644 --- a/include/mg_procedure.h +++ b/include/mg_procedure.h @@ -876,6 +876,65 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory, struct mgp_vertex **result); +/// Creates label index for given label. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if label index already exists, result will be 0, otherwise 1. +enum mgp_error mgp_create_label_index(struct mgp_graph *graph, const char *label, int *result); + +/// Drop label index. +enum mgp_error mgp_drop_label_index(struct mgp_graph *graph, const char *label, int *result); + +/// List all label indices. +enum mgp_error mgp_list_all_label_indices(struct mgp_graph *graph, struct mgp_memory *memory, struct mgp_list **result); + +/// Creates label-property index for given label and propery. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if label property index already exists, result will be 0, otherwise 1. +enum mgp_error mgp_create_label_property_index(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// Drops label-property index for given label and propery. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping label property index failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_label_property_index(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// List all label+property indices. +enum mgp_error mgp_list_all_label_property_indices(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + +/// Creates existence constraint for given label and property. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if creating existence constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_create_existence_constraint(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// Drops existence constraint for given label and property. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping existence constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_existence_constraint(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// List all existence constraints. +enum mgp_error mgp_list_all_existence_constraints(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + +/// Creates unique constraint for given label and properties. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if creating unique constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_create_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties, + int *result); + +/// Drops unique constraint for given label and properties. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping unique constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties, + int *result); + +/// List all unique constraints +enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + /// Result is non-zero if the graph can be modified. /// If a graph is immutable, then vertices cannot be created or deleted, and all of the returned vertices will be /// immutable also. The same applies for edges. diff --git a/include/mgp.hpp b/include/mgp.hpp index bea3545bb..25e365e53 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -21,12 +21,10 @@ #include #include #include -#include -#include - -#include #include +#include #include +#include #include "_mgp.hpp" #include "mg_exceptions.hpp" @@ -1289,7 +1287,7 @@ class Value { std::string_view ValueString() const; std::string_view ValueString(); /// @pre Value type needs to be Type::List. - const List ValueList() const; + List ValueList() const; List ValueList(); /// @pre Value type needs to be Type::Map. const Map ValueMap() const; @@ -3651,7 +3649,7 @@ inline std::string_view Value::ValueString() { return mgp::value_get_string(ptr_); } -inline const List Value::ValueList() const { +inline List Value::ValueList() const { if (Type() != Type::List) { throw ValueException("Type of value is wrong: expected List."); } @@ -4279,9 +4277,77 @@ inline void AddParamsReturnsToProc(mgp_proc *proc, std::vector ¶m } } // namespace detail +inline bool CreateLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) { + return create_label_index(memgaph_graph, label.data()); +} + +inline bool DropLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) { + return drop_label_index(memgaph_graph, label.data()); +} + +inline List ListAllLabelIndices(mgp_graph *memgraph_graph) { + auto *label_indices = mgp::MemHandlerCallback(list_all_label_indices, memgraph_graph); + if (label_indices == nullptr) { + throw ValueException("Couldn't list all label indices"); + } + return List(label_indices); +} + +inline bool CreateLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label, + const std::string_view property) { + return create_label_property_index(memgaph_graph, label.data(), property.data()); +} + +inline bool DropLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label, + const std::string_view property) { + return drop_label_property_index(memgaph_graph, label.data(), property.data()); +} + +inline List ListAllLabelPropertyIndices(mgp_graph *memgraph_graph) { + auto *label_property_indices = mgp::MemHandlerCallback(list_all_label_property_indices, memgraph_graph); + if (label_property_indices == nullptr) { + throw ValueException("Couldn't list all label+property indices"); + } + return List(label_property_indices); +} + +inline bool CreateExistenceConstraint(mgp_graph *memgraph_graph, const std::string_view label, + const std::string_view property) { + return create_existence_constraint(memgraph_graph, label.data(), property.data()); +} + +inline bool DropExistenceConstraint(mgp_graph *memgraph_graph, const std::string_view label, + const std::string_view property) { + return drop_existence_constraint(memgraph_graph, label.data(), property.data()); +} + +inline List ListAllExistenceConstraints(mgp_graph *memgraph_graph) { + auto *existence_constraints = mgp::MemHandlerCallback(list_all_existence_constraints, memgraph_graph); + if (existence_constraints == nullptr) { + throw ValueException("Couldn't list all existence_constraints"); + } + return List(existence_constraints); +} + +inline bool CreateUniqueConstraint(mgp_graph *memgraph_graph, const std::string_view label, mgp_value *properties) { + return create_unique_constraint(memgraph_graph, label.data(), properties); +} + +inline bool DropUniqueConstraint(mgp_graph *memgraph_graph, const std::string_view label, mgp_value *properties) { + return drop_unique_constraint(memgraph_graph, label.data(), properties); +} + +inline List ListAllUniqueConstraints(mgp_graph *memgraph_graph) { + auto *unique_constraints = mgp::MemHandlerCallback(list_all_unique_constraints, memgraph_graph); + if (unique_constraints == nullptr) { + throw ValueException("Couldn't list all unique_constraints"); + } + return List(unique_constraints); +} + void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType proc_type, std::vector parameters, std::vector returns, mgp_module *module, - mgp_memory *memory) { + mgp_memory * /*memory*/) { auto *proc = (proc_type == ProcedureType::Read) ? mgp::module_add_read_procedure(module, name.data(), callback) : mgp::module_add_write_procedure(module, name.data(), callback); detail::AddParamsReturnsToProc(proc, parameters, returns); @@ -4289,7 +4355,7 @@ void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType pro void AddBatchProcedure(mgp_proc_cb callback, mgp_proc_initializer initializer, mgp_proc_cleanup cleanup, std::string_view name, ProcedureType proc_type, std::vector parameters, - std::vector returns, mgp_module *module, mgp_memory *memory) { + std::vector returns, mgp_module *module, mgp_memory * /*memory*/) { auto *proc = (proc_type == ProcedureType::Read) ? mgp::module_add_batch_read_procedure(module, name.data(), callback, initializer, cleanup) : mgp::module_add_batch_write_procedure(module, name.data(), callback, initializer, cleanup); diff --git a/query_modules/schema.cpp b/query_modules/schema.cpp index 436e00716..d5a657e98 100644 --- a/query_modules/schema.cpp +++ b/query_modules/schema.cpp @@ -10,18 +10,33 @@ // licenses/APL.txt. #include +#include "utils/string.hpp" + +#include namespace Schema { -/*NodeTypeProperties and RelTypeProperties constants*/ +constexpr std::string_view kStatusKept = "Kept"; +constexpr std::string_view kStatusCreated = "Created"; +constexpr std::string_view kStatusDropped = "Dropped"; constexpr std::string_view kReturnNodeType = "nodeType"; constexpr std::string_view kProcedureNodeType = "node_type_properties"; constexpr std::string_view kProcedureRelType = "rel_type_properties"; +constexpr std::string_view kProcedureAssert = "assert"; constexpr std::string_view kReturnLabels = "nodeLabels"; constexpr std::string_view kReturnRelType = "relType"; constexpr std::string_view kReturnPropertyName = "propertyName"; constexpr std::string_view kReturnPropertyType = "propertyTypes"; constexpr std::string_view kReturnMandatory = "mandatory"; +constexpr std::string_view kReturnLabel = "label"; +constexpr std::string_view kReturnKey = "key"; +constexpr std::string_view kReturnKeys = "keys"; +constexpr std::string_view kReturnUnique = "unique"; +constexpr std::string_view kReturnAction = "action"; +constexpr std::string_view kParameterIndices = "indices"; +constexpr std::string_view kParameterUniqueConstraints = "unique_constraints"; +constexpr std::string_view kParameterExistenceConstraints = "existence_constraints"; +constexpr std::string_view kParameterDropExisting = "drop_existing"; std::string TypeOf(const mgp::Type &type); @@ -35,6 +50,7 @@ void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, con void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); +void Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); } // namespace Schema /*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this @@ -92,21 +108,22 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t record.Insert(std::string(kReturnMandatory).c_str(), mandatory); } -void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { +void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, + mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); for (auto node : graph.Nodes()) { - std::string type = ""; + std::string type; mgp::List labels = mgp::List(); for (auto label : node.Labels()) { labels.AppendExtend(mgp::Value(label)); type += ":`" + std::string(label) + "`"; } - if (node.Properties().size() == 0) { + if (node.Properties().empty()) { auto record = record_factory.NewRecord(); ProcessPropertiesNode(record, type, labels, "", "", false); continue; @@ -126,16 +143,15 @@ void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_r } } -void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { +void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; - ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); for (auto rel : graph.Relationships()) { std::string type = ":`" + std::string(rel.Type()) + "`"; - if (rel.Properties().size() == 0) { + if (rel.Properties().empty()) { auto record = record_factory.NewRecord(); ProcessPropertiesRel(record, type, "", "", false); continue; @@ -155,29 +171,436 @@ void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_re } } +void InsertRecordForLabelIndex(const auto &record_factory, const std::string_view label, + const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), ""); + record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List()); + record.Insert(std::string(Schema::kReturnUnique).c_str(), false); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void InsertRecordForUniqueConstraint(const auto &record_factory, const std::string_view label, + const mgp::List &properties, const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), properties.ToString()); + record.Insert(std::string(Schema::kReturnKeys).c_str(), properties); + record.Insert(std::string(Schema::kReturnUnique).c_str(), true); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void InsertRecordForLabelPropertyIndexAndExistenceConstraint(const auto &record_factory, const std::string_view label, + const std::string_view property, + const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), property); + record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List({mgp::Value(property)})); + record.Insert(std::string(Schema::kReturnUnique).c_str(), false); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void ProcessCreatingLabelIndex(const std::string_view label, const std::set &existing_label_indices, + mgp_graph *memgraph_graph, const auto &record_factory) { + if (existing_label_indices.contains(label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusKept); + } else if (mgp::CreateLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated); + } +} + +template +void ProcessCreatingLabelPropertyIndexAndExistenceConstraint(const std::string_view label, + const std::string_view property, + const std::set &existing_collection, + const TFunc &func_creation, mgp_graph *memgraph_graph, + const auto &record_factory) { + const auto label_property_search_key = std::string(label) + ":" + std::string(property); + if (existing_collection.contains(label_property_search_key)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusKept); + } else if (func_creation(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusCreated); + } +} + +/// We collect properties for which index was created. +using AssertedIndices = std::set>; +AssertedIndices CreateIndicesForLabel(const std::string_view label, const mgp::Value &properties_val, + mgp_graph *memgraph_graph, const auto &record_factory, + const std::set &existing_label_indices, + const std::set &existing_label_property_indices) { + AssertedIndices asserted_indices; + if (!properties_val.IsList()) { + return {}; + } + if (const auto properties = properties_val.ValueList(); + properties.Empty() && mgp::CreateLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated); + asserted_indices.emplace(""); + } else { + std::for_each(properties.begin(), properties.end(), + [&label, &existing_label_indices, &existing_label_property_indices, &memgraph_graph, &record_factory, + &asserted_indices](const mgp::Value &property) { + if (!property.IsString()) { + return; + } + const auto property_str = property.ValueString(); + if (property_str.empty()) { + ProcessCreatingLabelIndex(label, existing_label_indices, memgraph_graph, record_factory); + asserted_indices.emplace(""); + } else { + ProcessCreatingLabelPropertyIndexAndExistenceConstraint( + label, property_str, existing_label_property_indices, mgp::CreateLabelPropertyIndex, + memgraph_graph, record_factory); + asserted_indices.emplace(property_str); + } + }); + } + return asserted_indices; +} + +void ProcessIndices(const mgp::Map &indices_map, mgp_graph *memgraph_graph, const auto &record_factory, + bool drop_existing) { + auto mgp_existing_label_indices = mgp::ListAllLabelIndices(memgraph_graph); + auto mgp_existing_label_property_indices = mgp::ListAllLabelPropertyIndices(memgraph_graph); + + std::set existing_label_indices; + std::transform(mgp_existing_label_indices.begin(), mgp_existing_label_indices.end(), + std::inserter(existing_label_indices, existing_label_indices.begin()), + [](const mgp::Value &index) { return index.ValueString(); }); + + std::set existing_label_property_indices; + std::transform(mgp_existing_label_property_indices.begin(), mgp_existing_label_property_indices.end(), + std::inserter(existing_label_property_indices, existing_label_property_indices.begin()), + [](const mgp::Value &index) { return index.ValueString(); }); + + std::set asserted_label_indices; + std::set asserted_label_property_indices; + + auto merge_label_property = [](const std::string &label, const std::string &property) { + return label + ":" + property; + }; + + for (const auto &index : indices_map) { + const std::string_view label = index.key; + const mgp::Value &properties_val = index.value; + + AssertedIndices asserted_indices_new = CreateIndicesForLabel( + label, properties_val, memgraph_graph, record_factory, existing_label_indices, existing_label_property_indices); + + if (!drop_existing) { + continue; + } + std::ranges::for_each(asserted_indices_new, [&asserted_label_indices, &asserted_label_property_indices, label, + &merge_label_property](const std::string &property) { + if (property.empty()) { + asserted_label_indices.emplace(label); + } else { + asserted_label_property_indices.emplace(merge_label_property(std::string(label), property)); + } + }); + } + + if (!drop_existing) { + return; + } + + std::set label_indices_to_drop; + std::ranges::set_difference(existing_label_indices, asserted_label_indices, + std::inserter(label_indices_to_drop, label_indices_to_drop.begin())); + + std::ranges::for_each(label_indices_to_drop, [memgraph_graph, &record_factory](const std::string_view label) { + if (mgp::DropLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusDropped); + } + }); + + std::set label_property_indices_to_drop; + std::ranges::set_difference(existing_label_property_indices, asserted_label_property_indices, + std::inserter(label_property_indices_to_drop, label_property_indices_to_drop.begin())); + + auto decouple_label_property = [](const std::string_view label_property) { + const auto label_size = label_property.find(':'); + const auto label = std::string(label_property.substr(0, label_size)); + const auto property = std::string(label_property.substr(label_size + 1)); + return std::make_pair(label, property); + }; + + std::ranges::for_each(label_property_indices_to_drop, [memgraph_graph, &record_factory, decouple_label_property]( + const std::string_view label_property) { + const auto [label, property] = decouple_label_property(label_property); + if (mgp::DropLabelPropertyIndex(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped); + } + }); +} + +using ExistenceConstraintsStorage = std::set; + +ExistenceConstraintsStorage CreateExistenceConstraintsForLabel( + const std::string_view label, const mgp::Value &properties_val, mgp_graph *memgraph_graph, + const auto &record_factory, const std::set &existing_existence_constraints) { + ExistenceConstraintsStorage asserted_existence_constraints; + if (!properties_val.IsList()) { + return asserted_existence_constraints; + } + + auto validate_property = [](const mgp::Value &property) -> bool { + return property.IsString() && !property.ValueString().empty(); + }; + + const auto &properties = properties_val.ValueList(); + std::for_each(properties.begin(), properties.end(), + [&label, &existing_existence_constraints, &asserted_existence_constraints, &memgraph_graph, + &record_factory, &validate_property](const mgp::Value &property) { + if (!validate_property(property)) { + return; + } + const std::string_view property_str = property.ValueString(); + asserted_existence_constraints.emplace(property_str); + ProcessCreatingLabelPropertyIndexAndExistenceConstraint( + label, property_str, existing_existence_constraints, mgp::CreateExistenceConstraint, + memgraph_graph, record_factory); + }); + return asserted_existence_constraints; +} + +void ProcessExistenceConstraints(const mgp::Map &existence_constraints_map, mgp_graph *memgraph_graph, + const auto &record_factory, bool drop_existing) { + auto mgp_existing_existence_constraints = mgp::ListAllExistenceConstraints(memgraph_graph); + std::set existing_existence_constraints; + std::transform(mgp_existing_existence_constraints.begin(), mgp_existing_existence_constraints.end(), + std::inserter(existing_existence_constraints, existing_existence_constraints.begin()), + [](const mgp::Value &constraint) { return constraint.ValueString(); }); + + auto merge_label_property = [](const std::string_view label, const std::string_view property) { + auto str = std::string(label) + ":"; + str += property; + return str; + }; + + ExistenceConstraintsStorage asserted_existence_constraints; + + for (const auto &existing_constraint : existence_constraints_map) { + const std::string_view label = existing_constraint.key; + const mgp::Value &properties_val = existing_constraint.value; + auto asserted_existence_constraints_new = CreateExistenceConstraintsForLabel( + label, properties_val, memgraph_graph, record_factory, existing_existence_constraints); + if (!drop_existing) { + continue; + } + + std::ranges::for_each(asserted_existence_constraints_new, [&asserted_existence_constraints, &merge_label_property, + label](const std::string_view property) { + asserted_existence_constraints.emplace(merge_label_property(label, property)); + }); + } + + if (!drop_existing) { + return; + } + + std::set existence_constraints_to_drop; + std::ranges::set_difference(existing_existence_constraints, asserted_existence_constraints, + std::inserter(existence_constraints_to_drop, existence_constraints_to_drop.begin())); + + auto decouple_label_property = [](const std::string_view label_property) { + const auto label_size = label_property.find(':'); + const auto label = std::string(label_property.substr(0, label_size)); + const auto property = std::string(label_property.substr(label_size + 1)); + return std::make_pair(label, property); + }; + + std::ranges::for_each(existence_constraints_to_drop, [&](const std::string_view label_property) { + const auto [label, property] = decouple_label_property(label_property); + if (mgp::DropExistenceConstraint(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped); + } + }); +} + +using AssertedUniqueConstraintsStorage = std::set>; +AssertedUniqueConstraintsStorage CreateUniqueConstraintsForLabel( + const std::string_view label, const mgp::Value &unique_props_nested, + const std::map &existing_unique_constraints, + mgp_graph *memgraph_graph, const auto &record_factory) { + AssertedUniqueConstraintsStorage asserted_unique_constraints; + if (!unique_props_nested.IsList()) { + return asserted_unique_constraints; + } + + auto validate_unique_constraint_props = [](const mgp::Value &properties) -> bool { + if (!properties.IsList()) { + return false; + } + const auto &properties_list = properties.ValueList(); + if (properties_list.Empty()) { + return false; + } + return std::all_of(properties_list.begin(), properties_list.end(), [](const mgp::Value &property) { + return property.IsString() && !property.ValueString().empty(); + }); + }; + + auto unique_constraint_exists = + [](const std::string_view label, const std::set &properties, + const std::map &existing_unique_constraints) -> bool { + auto iter = existing_unique_constraints.find(label); + if (iter == existing_unique_constraints.end()) { + return false; + } + return iter->second.find(properties) != iter->second.end(); + }; + + for (const auto unique_props_nested_list = unique_props_nested.ValueList(); + const auto &properties : unique_props_nested_list) { + if (!validate_unique_constraint_props(properties)) { + continue; + } + const auto properties_list = properties.ValueList(); + std::set properties_coll; + std::transform(properties_list.begin(), properties_list.end(), + std::inserter(properties_coll, properties_coll.begin()), + [](const mgp::Value &property) { return property.ValueString(); }); + + if (unique_constraint_exists(label, properties_coll, existing_unique_constraints)) { + InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusKept); + } else if (mgp::CreateUniqueConstraint(memgraph_graph, label, properties.ptr())) { + InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusCreated); + } + asserted_unique_constraints.emplace(std::move(properties_coll)); + } + return asserted_unique_constraints; +} + +void ProcessUniqueConstraints(const mgp::Map &unique_constraints_map, mgp_graph *memgraph_graph, + const auto &record_factory, bool drop_existing) { + auto mgp_existing_unique_constraints = mgp::ListAllUniqueConstraints(memgraph_graph); + // label-unique_constraints pair + std::map existing_unique_constraints; + for (const auto &constraint : mgp_existing_unique_constraints) { + auto constraint_list = constraint.ValueList(); + std::set properties; + for (int i = 1; i < constraint_list.Size(); i++) { + properties.emplace(constraint_list[i].ValueString()); + } + const std::string_view label = constraint_list[0].ValueString(); + auto [it, inserted] = existing_unique_constraints.try_emplace(label, AssertedUniqueConstraintsStorage{properties}); + if (!inserted) { + it->second.emplace(std::move(properties)); + } + } + + std::map asserted_unique_constraints; + + for (const auto &[label, unique_props_nested] : unique_constraints_map) { + auto asserted_unique_constraints_new = CreateUniqueConstraintsForLabel( + label, unique_props_nested, existing_unique_constraints, memgraph_graph, record_factory); + if (drop_existing) { + asserted_unique_constraints.emplace(label, std::move(asserted_unique_constraints_new)); + } + } + + if (!drop_existing) { + return; + } + + std::vector>> unique_constraints_to_drop; + + // Check for each label for we found existing constraint in the DB whether it was asserted. + // If no unique constraint was found with label, we can drop all unique constraints for this label. (if branch) + // If some unique constraint was found with label, we can drop only those unique constraints that were not asserted. + // (else branch.) + std::ranges::for_each(existing_unique_constraints, [&asserted_unique_constraints, &unique_constraints_to_drop]( + const auto &existing_label_unique_constraints) { + const auto &label = existing_label_unique_constraints.first; + const auto &existing_unique_constraints_for_label = existing_label_unique_constraints.second; + const auto &asserted_unique_constraints_for_label = asserted_unique_constraints.find(label); + if (asserted_unique_constraints_for_label == asserted_unique_constraints.end()) { + std::ranges::for_each( + std::make_move_iterator(existing_unique_constraints_for_label.begin()), + std::make_move_iterator(existing_unique_constraints_for_label.end()), + [&unique_constraints_to_drop, &label](std::set existing_unique_constraint_for_label) { + unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label)); + }); + } else { + const auto &asserted_unique_constraints_for_label_coll = asserted_unique_constraints_for_label->second; + std::ranges::for_each( + std::make_move_iterator(existing_unique_constraints_for_label.begin()), + std::make_move_iterator(existing_unique_constraints_for_label.end()), + [&unique_constraints_to_drop, &label, &asserted_unique_constraints_for_label_coll]( + std::set existing_unique_constraint_for_label) { + if (!asserted_unique_constraints_for_label_coll.contains(existing_unique_constraint_for_label)) { + unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label)); + } + }); + } + }); + std::ranges::for_each( + unique_constraints_to_drop, [memgraph_graph, &record_factory](const auto &label_unique_constraint) { + const auto &[label, unique_constraint] = label_unique_constraint; + + auto unique_constraint_list = mgp::List(); + std::ranges::for_each(unique_constraint, [&unique_constraint_list](const std::string_view &property) { + unique_constraint_list.AppendExtend(mgp::Value(property)); + }); + + if (mgp::DropUniqueConstraint(memgraph_graph, label, mgp::Value(unique_constraint_list).ptr())) { + InsertRecordForUniqueConstraint(record_factory, label, unique_constraint_list, Schema::kStatusDropped); + } + }); +} + +void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto record_factory = mgp::RecordFactory(result); + auto arguments = mgp::List(args); + auto indices_map = arguments[0].ValueMap(); + auto unique_constraints_map = arguments[1].ValueMap(); + auto existence_constraints_map = arguments[2].ValueMap(); + auto drop_existing = arguments[3].ValueBool(); + + ProcessIndices(indices_map, memgraph_graph, record_factory, drop_existing); + ProcessExistenceConstraints(existence_constraints_map, memgraph_graph, record_factory, drop_existing); + ProcessUniqueConstraints(unique_constraints_map, memgraph_graph, record_factory, drop_existing); +} + extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { try { mgp::MemoryDispatcherGuard guard{memory}; ; - AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read, - {}, - {mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}), - mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any), - mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)}, + AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {}, + {mgp::Return(Schema::kReturnNodeType, mgp::Type::String), + mgp::Return(Schema::kReturnLabels, {mgp::Type::List, mgp::Type::String}), + mgp::Return(Schema::kReturnPropertyName, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any), + mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)}, module, memory); - AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read, - {}, - {mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any), - mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)}, + AddProcedure(Schema::RelTypeProperties, Schema::kProcedureRelType, mgp::ProcedureType::Read, {}, + {mgp::Return(Schema::kReturnRelType, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyName, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any), + mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)}, module, memory); - + AddProcedure( + Schema::Assert, Schema::kProcedureAssert, mgp::ProcedureType::Read, + { + mgp::Parameter(Schema::kParameterIndices, {mgp::Type::Map, mgp::Type::Any}), + mgp::Parameter(Schema::kParameterUniqueConstraints, {mgp::Type::Map, mgp::Type::Any}), + mgp::Parameter(Schema::kParameterExistenceConstraints, {mgp::Type::Map, mgp::Type::Any}, + mgp::Value(mgp::Map{})), + mgp::Parameter(Schema::kParameterDropExisting, mgp::Type::Bool, mgp::Value(true)), + }, + {mgp::Return(Schema::kReturnLabel, mgp::Type::String), mgp::Return(Schema::kReturnKey, mgp::Type::String), + mgp::Return(Schema::kReturnKeys, {mgp::Type::List, mgp::Type::String}), + mgp::Return(Schema::kReturnUnique, mgp::Type::Bool), mgp::Return(Schema::kReturnAction, mgp::Type::String)}, + module, memory); } catch (const std::exception &e) { + std::cerr << "Error while initializing query module: " << e.what() << std::endl; return 1; } diff --git a/src/query/db_accessor.cpp b/src/query/db_accessor.cpp index 0250ab695..df3fb808a 100644 --- a/src/query/db_accessor.cpp +++ b/src/query/db_accessor.cpp @@ -139,6 +139,8 @@ std::optional SubgraphDbAccessor::FindVertex(storage::Gid gid, s query::Graph *SubgraphDbAccessor::getGraph() { return graph_; } +DbAccessor *SubgraphDbAccessor::GetAccessor() { return &db_accessor_; } + VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; } storage::Result SubgraphVertexAccessor::OutEdges(storage::View view) const { diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index d6114edaf..75ec1e9ae 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -694,6 +694,8 @@ class SubgraphDbAccessor final { std::optional FindVertex(storage::Gid gid, storage::View view); Graph *getGraph(); + + DbAccessor *GetAccessor(); }; } // namespace memgraph::query diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 30bb4eca2..5aad0ff07 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -147,6 +147,8 @@ void memgraph::query::CurrentDB::CleanupDBTransaction(bool abort) { namespace memgraph::query { +constexpr std::string_view kSchemaAssert = "SCHEMA.ASSERT"; + template constexpr auto kAlwaysFalse = false; @@ -3715,7 +3717,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, // TODO: ATM only a single database, will change when we have multiple database transactions bool could_commit = utils::Downcast(parsed_query.query) != nullptr; bool unique = utils::Downcast(parsed_query.query) != nullptr || - utils::Downcast(parsed_query.query) != nullptr; + utils::Downcast(parsed_query.query) != nullptr || + upper_case_query.find(kSchemaAssert) != std::string::npos; SetupDatabaseTransaction(could_commit, unique); } diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index 2a176f2ed..f87377ba5 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -2535,6 +2535,333 @@ mgp_error mgp_graph_get_vertex_by_id(mgp_graph *graph, mgp_vertex_id id, mgp_mem result); } +mgp_error mgp_create_label_index(mgp_graph *graph, const char *label, int *result) { + return WrapExceptions( + [graph, label]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [label_id](memgraph::query::DbAccessor *impl) { return impl->CreateIndex(label_id); }, + [label_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateIndex(label_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_label_index(mgp_graph *graph, const char *label, int *result) { + return WrapExceptions( + [graph, label]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [label_id](memgraph::query::DbAccessor *impl) { return impl->DropIndex(label_id); }, + [label_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropIndex(label_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_list_all_label_indices(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto index_res = std::visit( + memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllIndices().label; }, + [](memgraph::query::SubgraphDbAccessor *impl) { return impl->GetAccessor()->ListAllIndices().label; }}, + graph->impl); + if (const auto err = mgp_list_make_empty(index_res.size(), memory, result); err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of creating list"); + } + for (const auto &label : index_res) { + const auto label_id_str = std::visit([label](const auto *impl) { return impl->LabelToName(label); }, graph->impl); + + mgp_value *label_value = nullptr; + if (const auto err_str = mgp_value_make_string(label_id_str.c_str(), memory, &label_value); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of creating label value"); + } + if (const auto err_list = mgp_list_append_extend(*result, label_value); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of appending label value"); + } + mgp_value_destroy(label_value); + } + }); +} + +mgp_error mgp_create_label_property_index(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->CreateIndex(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateIndex(label_id, property_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_label_property_index(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->DropIndex(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropIndex(label_id, property_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error create_and_append_label_property_to_mgp_list(mgp_graph *graph, mgp_memory *memory, mgp_list **result, + const auto &label_property_pair) { + return WrapExceptions([graph, memory, result, &label_property_pair]() { + const auto label_id_str = std::visit( + [label_id = label_property_pair.first](const auto *impl) { return impl->LabelToName(label_id); }, graph->impl); + const auto property_id_str = std::visit( + [property_id = label_property_pair.second](const auto *impl) { return impl->PropertyToName(property_id); }, + graph->impl); + + // This is hack to avoid dealing with pairs + mgp_value *label_property = nullptr; + auto final_str = label_id_str + ":"; + final_str += property_id_str; + + if (const auto err_str = mgp_value_make_string(final_str.c_str(), memory, &label_property); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Creating a list of label+property pairs failed due to failure of creating label+property value"); + } + if (const auto err_list = mgp_list_append_extend(*result, label_property); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Creating a list of label-property pairs due to failure of appending label+property value"); + } + + mgp_value_destroy(label_property); + }); +} + +mgp_error mgp_list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllIndices().label_property; }, + [](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->ListAllIndices().label_property; + }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(index_res.size(), memory, result); err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label+property indices failed due to failure of creating list"); + } + + for (const auto &label_property_pair : index_res) { + if (const auto err = create_and_append_label_property_to_mgp_list(graph, memory, result, label_property_pair); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Listing all label+property indices failed due to failure of appending label+property value"); + } + } + }); +} + +mgp_error mgp_create_existence_constraint(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto exist_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->CreateExistenceConstraint(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateExistenceConstraint(label_id, property_id); + }}, + graph->impl); + return exist_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_existence_constraint(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto exist_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->DropExistenceConstraint(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropExistenceConstraint(label_id, property_id); + }}, + graph->impl); + return exist_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto constraint_res = + std::visit(memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllConstraints().existence; }, + [](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->ListAllConstraints().existence; + }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(constraint_res.size(), memory, result); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all existence constraints failed due to failure of creating a list"); + } + + for (const auto &label_property_pair : constraint_res) { + if (const auto err = create_and_append_label_property_to_mgp_list(graph, memory, result, label_property_pair); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Listing all existence constraints failed due to failure of appending label+property value"); + } + } + }); +} + +mgp_error mgp_create_unique_constraint(mgp_graph *graph, const char *label, mgp_value *properties, int *result) { + return WrapExceptions( + [graph, label, properties]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + std::set property_ids; + for (const auto &elem : properties->list_v->elems) { + property_ids.insert(std::visit( + [prop_str = elem.string_v](auto *impl) { return impl->NameToProperty(prop_str); }, graph->impl)); + } + + const auto unique_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_ids](memgraph::query::DbAccessor *impl) { + return impl->CreateUniqueConstraint(label_id, property_ids); + }, + [label_id, property_ids](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateUniqueConstraint(label_id, property_ids); + }}, + graph->impl); + return unique_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_unique_constraint(mgp_graph *graph, const char *label, mgp_value *properties, int *result) { + return WrapExceptions( + [graph, label, properties]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + std::set property_ids; + for (const auto &elem : properties->list_v->elems) { + property_ids.insert(std::visit( + [prop_str = elem.string_v](auto *impl) { return impl->NameToProperty(prop_str); }, graph->impl)); + } + + const auto unique_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_ids](memgraph::query::DbAccessor *impl) { + return impl->DropUniqueConstraint(label_id, property_ids); + }, + [label_id, property_ids](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropUniqueConstraint(label_id, property_ids); + }}, + graph->impl); + return unique_res == memgraph::storage::UniqueConstraints::DeletionStatus::SUCCESS ? 1 : 0; + }, + result); +} + +mgp_error mgp_list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto constraints_res = std::visit( + memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllConstraints().unique; }, + [](memgraph::query::SubgraphDbAccessor *impl) { return impl->GetAccessor()->ListAllConstraints().unique; }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(constraints_res.size(), memory, result); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a list"); + } + + for (const auto &label_properties_pair : constraints_res) { + const std::string label_id_str = + std::visit([label_id = label_properties_pair.first](const auto *impl) { return impl->LabelToName(label_id); }, + graph->impl); + const std::vector properties_str = std::visit( + [property_ids = label_properties_pair.second](const auto *impl) { + std::vector property_ids_str; + property_ids_str.reserve(property_ids.size()); + std::transform(property_ids.begin(), property_ids.end(), std::back_inserter(property_ids_str), + [impl](const auto &property_id) { return impl->PropertyToName(property_id); }); + return property_ids_str; + }, + graph->impl); + + mgp_list *label_properties_mgp_list = nullptr; + if (const auto properties_mgp_list_err = + mgp_list_make_empty(properties_str.size() + 1, memory, &label_properties_mgp_list); + properties_mgp_list_err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating an inner list"); + } + + mgp_value *mgp_value_label = nullptr; + if (const auto err_label = mgp_value_make_string(label_id_str.c_str(), memory, &mgp_value_label); + err_label != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a label value"); + } + if (const auto err_label_into_list = mgp_list_append_extend(label_properties_mgp_list, mgp_value_label); + err_label_into_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of appending a label value"); + } + + mgp_value_destroy(mgp_value_label); + + for (const std::string &property_str : properties_str) { + mgp_value *property_mgp_value = nullptr; + if (const auto err_str = mgp_value_make_string(property_str.c_str(), memory, &property_mgp_value); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a property value"); + } + if (const auto err_list = mgp_list_append_extend(label_properties_mgp_list, property_mgp_value); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of appending a property value"); + } + mgp_value_destroy(property_mgp_value); + } + mgp_value value(label_properties_mgp_list, label_properties_mgp_list->GetMemoryResource()); + + if (const auto err_list = mgp_list_append_extend(*result, &value); err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating label+property value"); + } + mgp_value_destroy(&value); + } + }); +} + mgp_error mgp_graph_is_mutable(mgp_graph *graph, int *result) { *result = MgpGraphIsMutable(*graph) ? 1 : 0; return mgp_error::MGP_ERROR_NO_ERROR; diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index d0d1dd071..9f00081f6 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -1003,7 +1003,7 @@ void InMemoryStorage::InMemoryAccessor::FinalizeTransaction() { } utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex(LabelId label) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating label index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); if (!mem_label_index->CreateIndex(label, in_memory->vertices_.access(), std::nullopt)) { @@ -1017,7 +1017,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex( LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating label-property index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_property_index = static_cast(in_memory->indices_.label_property_index_.get()); @@ -1031,7 +1031,7 @@ utils::BasicResult InMemoryStorage::InMemoryA } utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex(LabelId label) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping label index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); if (!mem_label_index->DropIndex(label)) { @@ -1045,7 +1045,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex( LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping label-property index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_property_index = static_cast(in_memory->indices_.label_property_index_.get()); @@ -1060,7 +1060,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating existence requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); if (existence_constraints->ConstraintExists(label, property)) { @@ -1078,7 +1078,7 @@ InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, Prop utils::BasicResult InMemoryStorage::InMemoryAccessor::DropExistenceConstraint(LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping existence constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); if (!existence_constraints->DropConstraint(label, property)) { @@ -1090,7 +1090,7 @@ InMemoryStorage::InMemoryAccessor::DropExistenceConstraint(LabelId label, Proper utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const std::set &properties) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating unique constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_unique_constraints = static_cast(in_memory->constraints_.unique_constraints_.get()); @@ -1107,7 +1107,7 @@ InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const s UniqueConstraints::DeletionStatus InMemoryStorage::InMemoryAccessor::DropUniqueConstraint( LabelId label, const std::set &properties) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping unique constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_unique_constraints = static_cast(in_memory->constraints_.unique_constraints_.get()); diff --git a/tests/e2e/query_modules/schema_test.py b/tests/e2e/query_modules/schema_test.py index e61d9df1b..515514a74 100644 --- a/tests/e2e/query_modules/schema_test.py +++ b/tests/e2e/query_modules/schema_test.py @@ -15,6 +15,410 @@ import pytest from common import connect, execute_and_fetch_all +def test_assert_creates_label_index_empty_list(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: []}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_creates_label_index_empty_string(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_index_wrong_properties_type(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ''}, {}) YIELD * RETURN *;", + ) + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + + +def test_assert_property_is_not_a_string(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['name', 1]}, {}) YIELD * RETURN *;", + ) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label+property", "Person", "name", 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + + +def test_assert_creates_label_index_multiple_empty_strings(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', '', '', '']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_creates_label_property_index(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['name']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "name", ["name"], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label+property", "Person", "name", 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + + +def test_assert_creates_multiple_indices(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id', 'name'], Ball: ['', 'size', 'size', '']}, {}) YIELD * RETURN *;", + ) + ) + assert len(results) == 5 + assert results[0] == ("Created", "", [], "Ball", False) + assert results[1] == ("Created", "size", ["size"], "Ball", False) + assert results[2] == ("Created", "", [], "Person", False) + assert results[3] == ("Created", "id", ["id"], "Person", False) + assert results[4] == ("Created", "name", ["name"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert len(show_index_results) == 5 + assert show_index_results[0] == ("label", "Ball", None, 0) + assert show_index_results[1] == ("label", "Person", None, 0) + assert show_index_results[2] == ("label+property", "Ball", "size", 0) + assert show_index_results[3] == ("label+property", "Person", "id", 0) + assert show_index_results[4] == ("label+property", "Person", "name", 0) + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + execute_and_fetch_all(cursor, "DROP INDEX ON :Ball;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Ball(size);") + + +def test_assert_creates_existence_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert results == [ + ("Created", "name", ["name"], "Person", False), + ("Created", "surname", ["surname"], "Person", False), + ] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name"), ("exists", "Person", "surname")] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_dropping_indices(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(name);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Ball(size);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Ball;") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}) YIELD * RETURN *;")) + assert len(results) == 4 + assert results[0] == ("Dropped", "", [], "Ball", False) + assert results[1] == ("Dropped", "size", ["size"], "Ball", False) + assert results[2] == ("Dropped", "id", ["id"], "Person", False) + assert results[3] == ("Dropped", "name", ["name"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [] + + +def test_assert_existence_constraint_properties_not_list(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: 'name'}) YIELD * RETURN *;") + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + + +def test_assert_existence_constraint_property_not_string(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: ['name', 1]}) YIELD * RETURN *;") + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name")] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + + +def test_assert_existence_constraint_property_empty_string(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: ['']}) YIELD * RETURN *;") + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + + +def test_assert_creates_indices_and_existence_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id']}, {}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert len(results) == 4 + assert results[0] == ("Created", "", [], "Person", False) + assert results[1] == ("Created", "id", ["id"], "Person", False) + assert results[2] == ("Created", "name", ["name"], "Person", False) + assert results[3] == ("Created", "surname", ["surname"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name"), ("exists", "Person", "surname")] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_drops_existence_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 2 + assert results[0] == ("Dropped", "name", ["name"], "Person", False) + assert results[1] == ("Dropped", "surname", ["surname"], "Person", False) + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_creates_unique_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname']]}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_multiple_unique_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname'], ['id']]}) YIELD * RETURN *;", + ) + ) + assert results == [ + ("Created", "[name, surname]", ["name", "surname"], "Person", True), + ("Created", "[id]", ["id"], "Person", True), + ] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"]), ("unique", "Person", ["id"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + + +def test_assert_creates_unique_constraints_skip_invalid(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname'], 'wrong_type']}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_unique_constraints_skip_invalid_map_type(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname']], Ball: 'wrong_type'}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_constraints_and_indices(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id']}, {Person: [['name', 'surname'], ['id']]}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert len(results) == 6 + assert results[0] == ("Created", "", [], "Person", False) + assert results[1] == ("Created", "id", ["id"], "Person", False) + assert results[2] == ("Created", "name", ["name"], "Person", False) + assert results[3] == ("Created", "surname", ["surname"], "Person", False) + assert results[4] == ("Created", "[name, surname]", ["name", "surname"], "Person", True) + assert results[5] == ("Created", "[id]", ["id"], "Person", True) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [ + ("exists", "Person", "name"), + ("exists", "Person", "surname"), + ("unique", "Person", ["name", "surname"]), + ("unique", "Person", ["id"]), + ] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_drops_unique_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 2 + assert results[0] == ("Dropped", "[id]", ["id"], "Person", True) + assert results[1] == ("Dropped", "[name, surname]", ["name", "surname"], "Person", True) + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_drops_indices_and_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 6 + assert results[0] == ("Dropped", "", [], "Person", False) + assert results[1] == ("Dropped", "id", ["id"], "Person", False) + assert results[2] == ("Dropped", "name", ["name"], "Person", False) + assert results[3] == ("Dropped", "surname", ["surname"], "Person", False) + assert results[4] == ("Dropped", "[id]", ["id"], "Person", True) + assert results[5] == ("Dropped", "[name, surname]", ["name", "surname"], "Person", True) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_does_not_drop_indices_and_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}, false) YIELD * RETURN *;")) + assert len(results) == 0 + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [ + ("exists", "Person", "name"), + ("exists", "Person", "surname"), + ("unique", "Person", ["name", "surname"]), + ("unique", "Person", ["id"]), + ] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_keeps_existing_indices_and_constraints(): + cursor = connect().cursor() + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['id']}, {Person: [['name', 'surname']]}, {Person: ['name']}) YIELD * RETURN *;", + ) + ) + + print(results) + + assert len(results) == 6 + + assert results[0] == ("Kept", "id", ["id"], "Person", False) # label+property index on Person(id) should be kept + assert results[1] == ("Dropped", "", [], "Person", False) # label index on Person should be deleted + assert results[2] == ( + "Kept", + "name", + ["name"], + "Person", + False, + ) # existence constraint on Person(name) should be kept + assert results[3] == ( + "Dropped", + "surname", + ["surname"], + "Person", + False, + ) # existence constraint on surname should be deleted + assert results[4] == ( + "Kept", + "[name, surname]", + ["name", "surname"], + "Person", + True, + ) # unique constraint on Person(name, surname) should be kept + assert results[5] == ( + "Dropped", + "[id]", + ["id"], + "Person", + True, + ) # unique constraint on Person(id) should be deleted + + def test_node_type_properties1(): cursor = connect().cursor() execute_and_fetch_all( diff --git a/tests/e2e/query_modules/workloads.yaml b/tests/e2e/query_modules/workloads.yaml index ae6c45c90..54641c3d4 100644 --- a/tests/e2e/query_modules/workloads.yaml +++ b/tests/e2e/query_modules/workloads.yaml @@ -27,7 +27,7 @@ workloads: args: ["query_modules/mgps_test.py"] <<: *in_memory_cluster - - name: "Convert query module test" + - name: "Schema test" pre_set_workload: "tests/e2e/x.sh" binary: "tests/e2e/pytest_runner.sh" proc: "query_modules/" From e4f94c15c6ef9dcee26d98c977744331b51db4a1 Mon Sep 17 00:00:00 2001 From: Gareth Andrew Lloyd Date: Wed, 22 Nov 2023 13:05:02 +0000 Subject: [PATCH 24/52] Fixes for clang-tidy / sonar issues (#1536) --- include/mgp.hpp | 292 +++++++++--------- query_modules/schema.cpp | 2 - src/audit/log.cpp | 5 +- src/audit/log.hpp | 2 +- src/auth/models.cpp | 13 +- src/auth/models.hpp | 26 +- src/communication/bolt/client.hpp | 5 +- .../v1/decoder/chunked_decoder_buffer.hpp | 4 +- src/communication/bolt/v1/decoder/decoder.hpp | 6 +- .../bolt/v1/encoder/base_encoder.hpp | 10 +- .../v1/encoder/chunked_encoder_buffer.hpp | 4 +- .../bolt/v1/encoder/client_encoder.hpp | 2 +- src/communication/bolt/v1/encoder/encoder.hpp | 2 +- src/communication/bolt/v1/states/handlers.hpp | 6 +- src/communication/bolt/v1/states/init.hpp | 8 +- src/communication/bolt/v1/value.hpp | 2 +- src/communication/client.hpp | 4 +- src/communication/context.cpp | 6 +- src/communication/helpers.cpp | 4 +- src/communication/helpers.hpp | 4 +- src/communication/http/listener.hpp | 2 +- src/communication/listener.hpp | 5 +- src/communication/result_stream_faker.hpp | 4 +- src/communication/session.hpp | 4 +- src/communication/v2/listener.hpp | 2 +- src/communication/v2/session.hpp | 2 +- src/communication/websocket/server.hpp | 5 +- src/dbms/dbms_handler.hpp | 2 +- src/dbms/handler.hpp | 2 +- src/glue/SessionHL.cpp | 5 +- src/glue/SessionHL.hpp | 2 +- src/glue/communication.cpp | 4 +- src/helpers.hpp | 2 +- src/http_handlers/metrics.hpp | 1 - src/integrations/kafka/consumer.hpp | 4 +- src/integrations/pulsar/consumer.cpp | 4 +- src/io/network/endpoint.hpp | 10 +- src/io/network/epoll.hpp | 4 +- src/io/network/socket.hpp | 3 +- src/kvstore/kvstore.cpp | 2 +- src/query/common.hpp | 6 +- src/query/cypher_query_interpreter.hpp | 6 +- src/query/db_accessor.hpp | 2 - src/query/dump.cpp | 4 +- src/query/frame_change.hpp | 4 +- src/query/frontend/ast/ast.hpp | 15 +- src/query/frontend/ast/pretty_print.cpp | 2 +- src/query/frontend/opencypher/parser.hpp | 4 +- src/query/frontend/semantic/symbol.hpp | 19 +- .../frontend/semantic/symbol_generator.hpp | 2 +- src/query/frontend/semantic/symbol_table.hpp | 2 +- src/query/frontend/stripped.cpp | 37 +-- src/query/frontend/stripped.hpp | 4 +- .../frontend/stripped_lexer_constants.hpp | 8 +- src/query/interpret/eval.hpp | 4 +- src/query/interpreter.cpp | 31 +- src/query/interpreter.hpp | 2 +- src/query/plan/operator.cpp | 81 ++--- src/query/plan/operator.hpp | 111 ++++--- src/query/plan/planner.hpp | 8 +- src/query/plan/preprocess.cpp | 13 +- src/query/plan/preprocess.hpp | 48 +-- src/query/plan/profile.hpp | 7 +- src/query/plan/rewrite/index_lookup.hpp | 7 +- src/query/plan/variable_start_planner.cpp | 18 +- src/query/plan/variable_start_planner.hpp | 38 +-- src/query/plan/vertex_count_cache.hpp | 9 +- .../include/replication/config.hpp | 4 +- .../replication/replication_server.hpp | 4 +- src/requests/requests.cpp | 2 +- src/rpc/client.hpp | 2 +- src/rpc/client_pool.hpp | 7 +- src/rpc/exceptions.hpp | 2 +- src/rpc/protocol.cpp | 6 +- src/rpc/protocol.hpp | 2 +- src/slk/serialization.hpp | 20 +- src/slk/streams.cpp | 5 +- src/storage/v2/disk/label_property_index.cpp | 3 +- src/storage/v2/disk/rocksdb_storage.hpp | 2 +- src/storage/v2/disk/unique_constraints.cpp | 2 +- src/storage/v2/durability/serialization.hpp | 6 +- src/storage/v2/durability/snapshot.cpp | 4 +- src/storage/v2/durability/wal.hpp | 2 +- src/storage/v2/edge_accessor.hpp | 2 +- src/storage/v2/id_types.hpp | 2 +- src/storage/v2/modified_edge.hpp | 3 +- src/storage/v2/property_store.cpp | 8 +- .../v2/replication/replication_client.hpp | 2 +- src/storage/v2/replication/rpc.cpp | 7 +- src/storage/v2/replication/rpc.hpp | 38 +-- src/storage/v2/storage.hpp | 2 +- src/storage/v2/vertex_accessor.hpp | 2 +- src/telemetry/telemetry.cpp | 5 +- src/utils/cast.hpp | 10 +- src/utils/event_histogram.hpp | 7 +- src/utils/exceptions.hpp | 4 +- src/utils/logging.hpp | 22 +- src/utils/lru_cache.hpp | 2 +- src/utils/memory.hpp | 4 +- src/utils/scheduler.hpp | 2 +- src/utils/skip_list.hpp | 4 +- src/utils/small_vector.hpp | 42 +-- src/utils/stacktrace.hpp | 11 +- src/utils/variant_helpers.hpp | 2 +- tests/benchmark/query/profile.cpp | 14 +- tests/benchmark/rpc.cpp | 5 +- .../e2e/isolation_levels/isolation_levels.cpp | 2 +- .../memory/memory_limit_global_alloc_proc.cpp | 2 +- ..._limit_global_multi_thread_proc_create.cpp | 2 +- .../memory_limit_global_thread_alloc_proc.cpp | 2 +- tests/e2e/memory/procedure_memory_limit.cpp | 2 +- tests/e2e/memory/query_memory_limit_proc.cpp | 2 +- .../query_memory_limit_proc_multi_thread.cpp | 2 +- tests/e2e/replication/common.hpp | 4 +- tests/e2e/replication/constraints.cpp | 4 +- .../e2e/replication/read_write_benchmark.cpp | 4 +- tests/e2e/triggers/common.hpp | 4 +- tests/manual/antlr_tree_pretty_print.cpp | 4 +- tests/stress/long_running.cpp | 4 +- tests/unit/CMakeLists.txt | 17 +- tests/unit/auth.cpp | 20 +- tests/unit/auth_handler.cpp | 4 +- tests/unit/bfs_common.hpp | 2 +- tests/unit/bolt_chunked_encoder_buffer.cpp | 6 +- tests/unit/cpp_api.cpp | 2 +- tests/unit/cypher_main_visitor.cpp | 6 +- tests/unit/integrations_kafka_consumer.cpp | 6 +- tests/unit/interpreter.cpp | 4 +- tests/unit/kvstore.cpp | 6 +- tests/unit/mgp_kafka_c_api.cpp | 4 +- tests/unit/plan_pretty_print.cpp | 2 +- tests/unit/query_common.hpp | 22 +- tests/unit/query_cost_estimator.cpp | 12 +- tests/unit/query_dump.cpp | 8 +- tests/unit/query_expression_evaluator.cpp | 8 +- tests/unit/query_plan_checker.hpp | 17 +- tests/unit/query_plan_common.hpp | 14 +- tests/unit/query_plan_edge_cases.cpp | 4 +- tests/unit/query_plan_operator_to_string.cpp | 2 +- tests/unit/query_required_privileges.cpp | 2 +- tests/unit/query_semantic.cpp | 54 ++-- tests/unit/rpc_messages.hpp | 12 +- tests/unit/slk_core.cpp | 6 +- tests/unit/slk_streams.cpp | 4 +- tests/unit/storage_rocks.cpp | 2 +- tests/unit/storage_v2_property_store.cpp | 2 +- tests/unit/storage_v2_wal_file.cpp | 6 +- tests/unit/utils_file.cpp | 4 +- tests/unit/utils_string.cpp | 4 +- 149 files changed, 778 insertions(+), 763 deletions(-) diff --git a/include/mgp.hpp b/include/mgp.hpp index 25e365e53..6296d2e5c 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -34,7 +34,7 @@ namespace mgp { class IndexException : public std::exception { public: - explicit IndexException(const std::string &message) : message_(message) {} + explicit IndexException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -43,7 +43,7 @@ class IndexException : public std::exception { class ValueException : public std::exception { public: - explicit ValueException(const std::string &message) : message_(message) {} + explicit ValueException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -52,7 +52,7 @@ class ValueException : public std::exception { class NotFoundException : public std::exception { public: - explicit NotFoundException(const std::string &message) : message_(message) {} + explicit NotFoundException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -61,7 +61,7 @@ class NotFoundException : public std::exception { class MustAbortException : public std::exception { public: - explicit MustAbortException(const std::string &message) : message_(message) {} + explicit MustAbortException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -233,14 +233,14 @@ class Graph { GraphRelationships Relationships() const; /// @brief Returns the graph node with the given ID. - Node GetNodeById(const Id node_id) const; + Node GetNodeById(Id node_id) const; /// @brief Returns whether the graph contains a node with the given ID. - bool ContainsNode(const Id node_id) const; + bool ContainsNode(Id node_id) const; /// @brief Returns whether the graph contains the given node. bool ContainsNode(const Node &node) const; /// @brief Returns whether the graph contains a relationship with the given ID. - bool ContainsRelationship(const Id relationship_id) const; + bool ContainsRelationship(Id relationship_id) const; /// @brief Returns whether the graph contains the given relationship. bool ContainsRelationship(const Relationship &relationship) const; @@ -253,7 +253,7 @@ class Graph { /// @brief Deletes a node and all its incident edges from the graph. void DetachDeleteNode(const Node &node); /// @brief Creates a relationship of type `type` between nodes `from` and `to` and adds it to the graph. - Relationship CreateRelationship(const Node &from, const Node &to, const std::string_view type); + Relationship CreateRelationship(const Node &from, const Node &to, std::string_view type); /// @brief Changes a relationship from node. void SetFrom(Relationship &relationship, const Node &new_from); /// @brief Changes a relationship to node. @@ -305,7 +305,7 @@ class Nodes { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Node operator*() const; + Node operator*() const; private: mgp_vertices_iterator *nodes_iterator_ = nullptr; @@ -352,7 +352,7 @@ class GraphRelationships { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Relationship operator*() const; + Relationship operator*() const; private: mgp_vertices_iterator *nodes_iterator_ = nullptr; @@ -398,7 +398,7 @@ class Relationships { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Relationship operator*() const; + Relationship operator*() const; private: mgp_edges_iterator *relationships_iterator_ = nullptr; @@ -451,7 +451,7 @@ class Labels { Iterator &operator++(); - const std::string_view operator*() const; + std::string_view operator*() const; private: Iterator(const Labels *iterable, size_t index); @@ -502,7 +502,7 @@ class List { explicit List(std::vector &&values); /// @brief Creates a List from the given initializer_list. - explicit List(const std::initializer_list list); + explicit List(std::initializer_list list); List(const List &other) noexcept; List(List &&other) noexcept; @@ -518,7 +518,7 @@ class List { bool Empty() const; /// @brief Returns the value at the given `index`. - const Value operator[](size_t index) const; + Value operator[](size_t index) const; ///@brief Same as above, but non const value Value operator[](size_t index); @@ -540,7 +540,7 @@ class List { Iterator &operator++(); - const Value operator*() const; + Value operator*() const; private: Iterator(const List *iterable, size_t index); @@ -577,7 +577,7 @@ class List { bool operator!=(const List &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_list *ptr_; @@ -608,7 +608,7 @@ class Map { explicit Map(std::map &&items); /// @brief Creates a Map from the given initializer_list (map items correspond to initializer list pairs). - Map(const std::initializer_list> items); + Map(std::initializer_list> items); Map(const Map &other) noexcept; Map(Map &&other) noexcept; @@ -625,10 +625,10 @@ class Map { bool Empty() const; /// @brief Returns the value at the given `key`. - Value const operator[](std::string_view key) const; + Value operator[](std::string_view key) const; /// @brief Returns the value at the given `key`. - Value const At(std::string_view key) const; + Value At(std::string_view key) const; /// @brief Returns true if the given `key` exists. bool KeyExists(std::string_view key) const; @@ -656,7 +656,7 @@ class Map { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const MapItem operator*() const; + MapItem operator*() const; private: mgp_map_items_iterator *map_items_iterator_ = nullptr; @@ -698,7 +698,7 @@ class Map { bool operator!=(const Map &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_map *ptr_; @@ -761,10 +761,10 @@ class Node { Relationships OutRelationships() const; /// @brief Adds a label to the node. - void AddLabel(const std::string_view label); + void AddLabel(std::string_view label); /// @brief Removes a label from the node. - void RemoveLabel(const std::string_view label); + void RemoveLabel(std::string_view label); bool operator<(const Node &other) const; @@ -775,7 +775,7 @@ class Node { bool operator!=(const Node &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; /// @brief returns the in degree of a node inline size_t InDegree() const; @@ -845,7 +845,7 @@ class Relationship { bool operator!=(const Relationship &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_edge *ptr_; @@ -898,7 +898,7 @@ class Path { bool operator!=(const Path &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_path *ptr_; @@ -958,7 +958,7 @@ class Date { bool operator<(const Date &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_date *ptr_; @@ -1020,7 +1020,7 @@ class LocalTime { bool operator<(const LocalTime &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_local_time *ptr_; @@ -1088,7 +1088,7 @@ class LocalDateTime { bool operator<(const LocalDateTime &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_local_date_time *ptr_; @@ -1142,7 +1142,7 @@ class Duration { bool operator<(const Duration &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_duration *ptr_; @@ -1190,13 +1190,13 @@ class Value { explicit Value(); // Primitive type constructors: - explicit Value(const bool value); - explicit Value(const int64_t value); - explicit Value(const double value); + explicit Value(bool value); + explicit Value(int64_t value); + explicit Value(double value); // String constructors: explicit Value(const char *value); - explicit Value(const std::string_view value); + explicit Value(std::string_view value); // Container constructors: /// @brief Constructs a List value from the copy of the given `list`. @@ -1290,28 +1290,28 @@ class Value { List ValueList() const; List ValueList(); /// @pre Value type needs to be Type::Map. - const Map ValueMap() const; + Map ValueMap() const; Map ValueMap(); /// @pre Value type needs to be Type::Node. - const Node ValueNode() const; + Node ValueNode() const; Node ValueNode(); /// @pre Value type needs to be Type::Relationship. - const Relationship ValueRelationship() const; + Relationship ValueRelationship() const; Relationship ValueRelationship(); /// @pre Value type needs to be Type::Path. - const Path ValuePath() const; + Path ValuePath() const; Path ValuePath(); /// @pre Value type needs to be Type::Date. - const Date ValueDate() const; + Date ValueDate() const; Date ValueDate(); /// @pre Value type needs to be Type::LocalTime. - const LocalTime ValueLocalTime() const; + LocalTime ValueLocalTime() const; LocalTime ValueLocalTime(); /// @pre Value type needs to be Type::LocalDateTime. - const LocalDateTime ValueLocalDateTime() const; + LocalDateTime ValueLocalDateTime() const; LocalDateTime ValueLocalDateTime(); /// @pre Value type needs to be Type::Duration. - const Duration ValueDuration() const; + Duration ValueDuration() const; Duration ValueDuration(); /// @brief Returns whether the value is null. @@ -1355,7 +1355,7 @@ class Value { friend std::ostream &operator<<(std::ostream &os, const mgp::Value &value); /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_value *ptr_; @@ -1421,9 +1421,9 @@ class RecordFactory { public: explicit RecordFactory(mgp_result *result); - const Record NewRecord() const; + Record NewRecord() const; - void SetErrorMessage(const std::string_view error_msg) const; + void SetErrorMessage(std::string_view error_msg) const; void SetErrorMessage(const char *error_msg) const; @@ -1465,7 +1465,7 @@ class Result { /// @brief Sets a @ref Duration value to be returned. inline void SetValue(const Duration &duration); - void SetErrorMessage(const std::string_view error_msg) const; + void SetErrorMessage(std::string_view error_msg) const; void SetErrorMessage(const char *error_msg) const; @@ -1644,8 +1644,8 @@ template TDest MemcpyCast(TSrc src) { TDest dest; static_assert(sizeof(dest) == sizeof(src), "MemcpyCast expects source and destination to be of the same size"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects source to be an arithmetic type"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects destination to be an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects source to be an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects destination to be an arithmetic type"); std::memcpy(&dest, &src, sizeof(src)); return dest; } @@ -1678,8 +1678,8 @@ inline bool MapsEqual(mgp_map *map1, mgp_map *map2) { if (mgp::map_size(map1) != mgp::map_size(map2)) { return false; } - auto items_it = mgp::MemHandlerCallback(map_iter_items, map1); - for (auto item = mgp::map_items_iterator_get(items_it); item; item = mgp::map_items_iterator_next(items_it)) { + auto *items_it = mgp::MemHandlerCallback(map_iter_items, map1); + for (auto *item = mgp::map_items_iterator_get(items_it); item; item = mgp::map_items_iterator_next(items_it)) { if (mgp::map_item_key(item) == mgp::map_item_key(item)) { return false; } @@ -1943,7 +1943,7 @@ inline int64_t Graph::Size() const { } inline GraphNodes Graph::Nodes() const { - auto nodes_it = mgp::MemHandlerCallback(graph_iter_vertices, graph_); + auto *nodes_it = mgp::MemHandlerCallback(graph_iter_vertices, graph_); if (nodes_it == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -1953,7 +1953,7 @@ inline GraphNodes Graph::Nodes() const { inline GraphRelationships Graph::Relationships() const { return GraphRelationships(graph_); } inline Node Graph::GetNodeById(const Id node_id) const { - auto mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); + auto *mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); if (mgp_node == nullptr) { mgp::vertex_destroy(mgp_node); throw NotFoundException("Node with ID " + std::to_string(node_id.AsUint()) + " not found!"); @@ -1964,7 +1964,7 @@ inline Node Graph::GetNodeById(const Id node_id) const { } inline bool Graph::ContainsNode(const Id node_id) const { - auto mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); + auto *mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); if (mgp_node == nullptr) { return false; } @@ -2066,7 +2066,7 @@ inline Nodes::Iterator::~Iterator() { inline Nodes::Iterator &Nodes::Iterator::operator++() { if (nodes_iterator_ != nullptr) { - auto next = mgp::vertices_iterator_next(nodes_iterator_); + auto *next = mgp::vertices_iterator_next(nodes_iterator_); if (next == nullptr) { mgp::vertices_iterator_destroy(nodes_iterator_); @@ -2098,7 +2098,7 @@ inline bool Nodes::Iterator::operator==(Iterator other) const { inline bool Nodes::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Node Nodes::Iterator::operator*() const { +inline Node Nodes::Iterator::operator*() const { if (nodes_iterator_ == nullptr) { return Node((const mgp_vertex *)nullptr); } @@ -2126,7 +2126,7 @@ inline GraphRelationships::Iterator::Iterator(mgp_vertices_iterator *nodes_itera } // Go through each graph node’s adjacent nodes - for (auto node = mgp::vertices_iterator_get(nodes_iterator_); node; + for (auto *node = mgp::vertices_iterator_get(nodes_iterator_); node; node = mgp::vertices_iterator_next(nodes_iterator_)) { // Check if node exists if (node == nullptr) { @@ -2137,7 +2137,7 @@ inline GraphRelationships::Iterator::Iterator(mgp_vertices_iterator *nodes_itera // Check if node has out-relationships out_relationships_iterator_ = mgp::MemHandlerCallback(vertex_iter_out_edges, node); - auto relationship = mgp::edges_iterator_get(out_relationships_iterator_); + auto *relationship = mgp::edges_iterator_get(out_relationships_iterator_); if (relationship != nullptr) { return; } @@ -2164,7 +2164,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // 1. Check if the current node has remaining relationships to iterate over if (out_relationships_iterator_ != nullptr) { - auto next = mgp::edges_iterator_next(out_relationships_iterator_); + auto *next = mgp::edges_iterator_next(out_relationships_iterator_); if (next != nullptr) { return *this; @@ -2177,7 +2177,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // 2. Move onto the next nodes if (nodes_iterator_ != nullptr) { - for (auto node = mgp::vertices_iterator_next(nodes_iterator_); node; + for (auto *node = mgp::vertices_iterator_next(nodes_iterator_); node; node = mgp::vertices_iterator_next(nodes_iterator_)) { // Check if node exists - if it doesn’t, we’ve reached the end of the iterator if (node == nullptr) { @@ -2188,7 +2188,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // Check if node has out-relationships out_relationships_iterator_ = mgp::MemHandlerCallback(vertex_iter_out_edges, node); - auto relationship = mgp::edges_iterator_get(out_relationships_iterator_); + auto *relationship = mgp::edges_iterator_get(out_relationships_iterator_); if (relationship != nullptr) { return *this; } @@ -2222,7 +2222,7 @@ inline bool GraphRelationships::Iterator::operator==(Iterator other) const { inline bool GraphRelationships::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Relationship GraphRelationships::Iterator::operator*() const { +inline Relationship GraphRelationships::Iterator::operator*() const { if (out_relationships_iterator_ != nullptr) { return Relationship(mgp::edges_iterator_get(out_relationships_iterator_)); } @@ -2268,7 +2268,7 @@ inline Relationships::Iterator::~Iterator() { inline Relationships::Iterator &Relationships::Iterator::operator++() { if (relationships_iterator_ != nullptr) { - auto next = mgp::edges_iterator_next(relationships_iterator_); + auto *next = mgp::edges_iterator_next(relationships_iterator_); if (next == nullptr) { mgp::edges_iterator_destroy(relationships_iterator_); @@ -2300,7 +2300,7 @@ inline bool Relationships::Iterator::operator==(Iterator other) const { inline bool Relationships::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Relationship Relationships::Iterator::operator*() const { +inline Relationship Relationships::Iterator::operator*() const { if (relationships_iterator_ == nullptr) { return Relationship((mgp_edge *)nullptr); } @@ -2361,7 +2361,7 @@ inline Labels::Iterator &Labels::Iterator::operator++() { return *this; } -inline const std::string_view Labels::Iterator::operator*() const { return (*iterable_)[index_]; } +inline std::string_view Labels::Iterator::operator*() const { return (*iterable_)[index_]; } inline Labels::Iterator::Iterator(const Labels *iterable, size_t index) : iterable_(iterable), index_(index) {} @@ -2446,7 +2446,7 @@ inline size_t List::Size() const { return mgp::list_size(ptr_); } inline bool List::Empty() const { return Size() == 0; } -inline const Value List::operator[](size_t index) const { return Value(mgp::list_at(ptr_, index)); } +inline Value List::operator[](size_t index) const { return Value(mgp::list_at(ptr_, index)); } inline Value List::operator[](size_t index) { return Value(mgp::list_at(ptr_, index)); } @@ -2461,7 +2461,7 @@ inline List::Iterator &List::Iterator::operator++() { return *this; } -inline const Value List::Iterator::operator*() const { return (*iterable_)[index_]; } +inline Value List::Iterator::operator*() const { return (*iterable_)[index_]; } inline List::Iterator::Iterator(const List *iterable, size_t index) : iterable_(iterable), index_(index) {} @@ -2488,7 +2488,7 @@ inline bool List::operator==(const List &other) const { return util::ListsEqual( inline bool List::operator!=(const List &other) const { return !(*this == other); } -inline const std::string List::ToString() const { +inline std::string List::ToString() const { const size_t size = Size(); if (size == 0) { return "[]"; @@ -2572,9 +2572,9 @@ inline size_t Map::Size() const { return mgp::map_size(ptr_); } inline bool Map::Empty() const { return Size() == 0; } -inline const Value Map::operator[](std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); } +inline Value Map::operator[](std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); } -inline const Value Map::At(std::string_view key) const { +inline Value Map::At(std::string_view key) const { auto *ptr = mgp::map_at(ptr_, key.data()); if (ptr) { return Value(ptr); @@ -2603,7 +2603,7 @@ inline Map::Iterator::~Iterator() { inline Map::Iterator &Map::Iterator::operator++() { if (map_items_iterator_ != nullptr) { - auto next = mgp::map_items_iterator_next(map_items_iterator_); + auto *next = mgp::map_items_iterator_next(map_items_iterator_); if (next == nullptr) { mgp::map_items_iterator_destroy(map_items_iterator_); @@ -2632,14 +2632,14 @@ inline bool Map::Iterator::operator==(Iterator other) const { inline bool Map::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const MapItem Map::Iterator::operator*() const { +inline MapItem Map::Iterator::operator*() const { if (map_items_iterator_ == nullptr) { throw ValueException("Empty map item!"); } - auto raw_map_item = mgp::map_items_iterator_get(map_items_iterator_); + auto *raw_map_item = mgp::map_items_iterator_get(map_items_iterator_); - auto map_key = mgp::map_item_key(raw_map_item); + const auto *map_key = mgp::map_item_key(raw_map_item); auto map_value = Value(mgp::map_item_value(raw_map_item)); return MapItem{.key = map_key, .value = map_value}; @@ -2675,7 +2675,7 @@ inline bool Map::operator==(const Map &other) const { return util::MapsEqual(ptr inline bool Map::operator!=(const Map &other) const { return !(*this == other); } -inline const std::string Map::ToString() const { +inline std::string Map::ToString() const { const size_t map_size = Size(); if (map_size == 0) { return "{}"; @@ -2747,7 +2747,7 @@ inline bool Node::HasLabel(std::string_view label) const { } inline Relationships Node::InRelationships() const { - auto relationship_iterator = mgp::MemHandlerCallback(vertex_iter_in_edges, ptr_); + auto *relationship_iterator = mgp::MemHandlerCallback(vertex_iter_in_edges, ptr_); if (relationship_iterator == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -2755,7 +2755,7 @@ inline Relationships Node::InRelationships() const { } inline Relationships Node::OutRelationships() const { - auto relationship_iterator = mgp::MemHandlerCallback(vertex_iter_out_edges, ptr_); + auto *relationship_iterator = mgp::MemHandlerCallback(vertex_iter_out_edges, ptr_); if (relationship_iterator == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -2825,7 +2825,7 @@ inline std::string PropertiesToString(const std::map &proper return properties; } -inline const std::string Node::ToString() const { +inline std::string Node::ToString() const { std::string labels{", "}; for (auto label : Labels()) { labels.append(":" + std::string(label)); @@ -2933,7 +2933,7 @@ inline bool Relationship::operator==(const Relationship &other) const { inline bool Relationship::operator!=(const Relationship &other) const { return !(*this == other); } -inline const std::string Relationship::ToString() const { +inline std::string Relationship::ToString() const { const auto from = From(); const auto to = To(); @@ -2992,7 +2992,7 @@ inline Path::~Path() { inline size_t Path::Length() const { return mgp::path_size(ptr_); } inline Node Path::GetNodeAt(size_t index) const { - auto node_ptr = mgp::path_vertex_at(ptr_, index); + auto *node_ptr = mgp::path_vertex_at(ptr_, index); if (node_ptr == nullptr) { throw IndexException("Index value out of bounds."); } @@ -3000,7 +3000,7 @@ inline Node Path::GetNodeAt(size_t index) const { } inline Relationship Path::GetRelationshipAt(size_t index) const { - auto relationship_ptr = mgp::path_edge_at(ptr_, index); + auto *relationship_ptr = mgp::path_edge_at(ptr_, index); if (relationship_ptr == nullptr) { throw IndexException("Index value out of bounds."); } @@ -3015,10 +3015,10 @@ inline bool Path::operator==(const Path &other) const { return util::PathsEqual( inline bool Path::operator!=(const Path &other) const { return !(*this == other); } -inline const std::string Path::ToString() const { +inline std::string Path::ToString() const { const auto length = Length(); size_t i = 0; - std::string return_string{""}; + std::string return_string; for (i = 0; i < length; i++) { const auto node = GetNodeAt(i); return_string.append(node.ToString() + "-"); @@ -3089,7 +3089,7 @@ inline Date::~Date() { } inline Date Date::Now() { - auto mgp_date = mgp::MemHandlerCallback(date_now); + auto *mgp_date = mgp::MemHandlerCallback(date_now); auto date = Date(mgp_date); mgp::date_destroy(mgp_date); @@ -3107,7 +3107,7 @@ inline int64_t Date::Timestamp() const { return mgp::date_timestamp(ptr_); } inline bool Date::operator==(const Date &other) const { return util::DatesEqual(ptr_, other.ptr_); } inline Date Date::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); auto sum = Date(mgp_sum); mgp::date_destroy(mgp_sum); @@ -3115,7 +3115,7 @@ inline Date Date::operator+(const Duration &dur) const { } inline Date Date::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); auto difference = Date(mgp_difference); mgp::date_destroy(mgp_difference); @@ -3123,7 +3123,7 @@ inline Date Date::operator-(const Duration &dur) const { } inline Duration Date::operator-(const Date &other) const { - auto mgp_difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3131,14 +3131,14 @@ inline Duration Date::operator-(const Date &other) const { } inline bool Date::operator<(const Date &other) const { - auto difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string Date::ToString() const { +inline std::string Date::ToString() const { return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()); } @@ -3188,7 +3188,7 @@ inline LocalTime::~LocalTime() { } inline LocalTime LocalTime::Now() { - auto mgp_local_time = mgp::MemHandlerCallback(local_time_now); + auto *mgp_local_time = mgp::MemHandlerCallback(local_time_now); auto local_time = LocalTime(mgp_local_time); mgp::local_time_destroy(mgp_local_time); @@ -3210,7 +3210,7 @@ inline int64_t LocalTime::Timestamp() const { return mgp::local_time_timestamp(p inline bool LocalTime::operator==(const LocalTime &other) const { return util::LocalTimesEqual(ptr_, other.ptr_); } inline LocalTime LocalTime::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(local_time_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(local_time_add_duration, ptr_, dur.ptr_); auto sum = LocalTime(mgp_sum); mgp::local_time_destroy(mgp_sum); @@ -3218,7 +3218,7 @@ inline LocalTime LocalTime::operator+(const Duration &dur) const { } inline LocalTime LocalTime::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(local_time_sub_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_time_sub_duration, ptr_, dur.ptr_); auto difference = LocalTime(mgp_difference); mgp::local_time_destroy(mgp_difference); @@ -3226,7 +3226,7 @@ inline LocalTime LocalTime::operator-(const Duration &dur) const { } inline Duration LocalTime::operator-(const LocalTime &other) const { - auto mgp_difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3234,14 +3234,14 @@ inline Duration LocalTime::operator-(const LocalTime &other) const { } inline bool LocalTime::operator<(const LocalTime &other) const { - auto difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string LocalTime::ToString() const { +inline std::string LocalTime::ToString() const { return std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + std::to_string(Millisecond()) + std::to_string(Microsecond()); } @@ -3299,7 +3299,7 @@ inline LocalDateTime::~LocalDateTime() { } inline LocalDateTime LocalDateTime::Now() { - auto mgp_local_date_time = mgp::MemHandlerCallback(local_date_time_now); + auto *mgp_local_date_time = mgp::MemHandlerCallback(local_date_time_now); auto local_date_time = LocalDateTime(mgp_local_date_time); mgp::local_date_time_destroy(mgp_local_date_time); @@ -3329,7 +3329,7 @@ inline bool LocalDateTime::operator==(const LocalDateTime &other) const { } inline LocalDateTime LocalDateTime::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(local_date_time_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(local_date_time_add_duration, ptr_, dur.ptr_); auto sum = LocalDateTime(mgp_sum); mgp::local_date_time_destroy(mgp_sum); @@ -3337,7 +3337,7 @@ inline LocalDateTime LocalDateTime::operator+(const Duration &dur) const { } inline LocalDateTime LocalDateTime::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(local_date_time_sub_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_date_time_sub_duration, ptr_, dur.ptr_); auto difference = LocalDateTime(mgp_difference); mgp::local_date_time_destroy(mgp_difference); @@ -3345,7 +3345,7 @@ inline LocalDateTime LocalDateTime::operator-(const Duration &dur) const { } inline Duration LocalDateTime::operator-(const LocalDateTime &other) const { - auto mgp_difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3353,14 +3353,14 @@ inline Duration LocalDateTime::operator-(const LocalDateTime &other) const { } inline bool LocalDateTime::operator<(const LocalDateTime &other) const { - auto difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string LocalDateTime::ToString() const { +inline std::string LocalDateTime::ToString() const { return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()) + "T" + std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + std::to_string(Millisecond()) + std::to_string(Microsecond()); @@ -3424,7 +3424,7 @@ inline int64_t Duration::Microseconds() const { return mgp::duration_get_microse inline bool Duration::operator==(const Duration &other) const { return util::DurationsEqual(ptr_, other.ptr_); } inline Duration Duration::operator+(const Duration &other) const { - auto mgp_sum = mgp::MemHandlerCallback(duration_add, ptr_, other.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(duration_add, ptr_, other.ptr_); auto sum = Duration(mgp_sum); mgp::duration_destroy(mgp_sum); @@ -3432,7 +3432,7 @@ inline Duration Duration::operator+(const Duration &other) const { } inline Duration Duration::operator-(const Duration &other) const { - auto mgp_difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3440,7 +3440,7 @@ inline Duration Duration::operator-(const Duration &other) const { } inline Duration Duration::operator-() const { - auto mgp_neg = mgp::MemHandlerCallback(duration_neg, ptr_); + auto *mgp_neg = mgp::MemHandlerCallback(duration_neg, ptr_); auto neg = Duration(mgp_neg); mgp::duration_destroy(mgp_neg); @@ -3448,14 +3448,14 @@ inline Duration Duration::operator-() const { } inline bool Duration::operator<(const Duration &other) const { - auto difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string Duration::ToString() const { return std::to_string(Microseconds()) + "ms"; } +inline std::string Duration::ToString() const { return std::to_string(Microseconds()) + "ms"; } /* #endregion */ @@ -3662,7 +3662,7 @@ inline List Value::ValueList() { return List(mgp::value_get_list(ptr_)); } -inline const Map Value::ValueMap() const { +inline Map Value::ValueMap() const { if (Type() != Type::Map) { throw ValueException("Type of value is wrong: expected Map."); } @@ -3675,7 +3675,7 @@ inline Map Value::ValueMap() { return Map(mgp::value_get_map(ptr_)); } -inline const Node Value::ValueNode() const { +inline Node Value::ValueNode() const { if (Type() != Type::Node) { throw ValueException("Type of value is wrong: expected Node."); } @@ -3688,7 +3688,7 @@ inline Node Value::ValueNode() { return Node(mgp::value_get_vertex(ptr_)); } -inline const Relationship Value::ValueRelationship() const { +inline Relationship Value::ValueRelationship() const { if (Type() != Type::Relationship) { throw ValueException("Type of value is wrong: expected Relationship."); } @@ -3701,7 +3701,7 @@ inline Relationship Value::ValueRelationship() { return Relationship(mgp::value_get_edge(ptr_)); } -inline const Path Value::ValuePath() const { +inline Path Value::ValuePath() const { if (Type() != Type::Path) { throw ValueException("Type of value is wrong: expected Path."); } @@ -3714,7 +3714,7 @@ inline Path Value::ValuePath() { return Path(mgp::value_get_path(ptr_)); } -inline const Date Value::ValueDate() const { +inline Date Value::ValueDate() const { if (Type() != Type::Date) { throw ValueException("Type of value is wrong: expected Date."); } @@ -3727,7 +3727,7 @@ inline Date Value::ValueDate() { return Date(mgp::value_get_date(ptr_)); } -inline const LocalTime Value::ValueLocalTime() const { +inline LocalTime Value::ValueLocalTime() const { if (Type() != Type::LocalTime) { throw ValueException("Type of value is wrong: expected LocalTime."); } @@ -3740,7 +3740,7 @@ inline LocalTime Value::ValueLocalTime() { return LocalTime(mgp::value_get_local_time(ptr_)); } -inline const LocalDateTime Value::ValueLocalDateTime() const { +inline LocalDateTime Value::ValueLocalDateTime() const { if (Type() != Type::LocalDateTime) { throw ValueException("Type of value is wrong: expected LocalDateTime."); } @@ -3753,7 +3753,7 @@ inline LocalDateTime Value::ValueLocalDateTime() { return LocalDateTime(mgp::value_get_local_date_time(ptr_)); } -inline const Duration Value::ValueDuration() const { +inline Duration Value::ValueDuration() const { if (Type() != Type::Duration) { throw ValueException("Type of value is wrong: expected Duration."); } @@ -3921,7 +3921,7 @@ inline std::ostream &operator<<(std::ostream &os, const mgp::Type &type) { } } -inline const std::string Value::ToString() const { +inline std::string Value::ToString() const { const mgp::Type &type = Type(); switch (type) { case Type::Null: @@ -3965,85 +3965,85 @@ inline const std::string Value::ToString() const { inline Record::Record(mgp_result_record *record) : record_(record) {} inline void Record::Insert(const char *field_name, bool value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_bool, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_bool, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, std::int64_t value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_int, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_int, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, double value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_double, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_double, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, std::string_view value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const char *value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const List &list) { - auto mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); + auto *mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Map &map) { - auto mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); + auto *mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Node &node) { - auto mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); + auto *mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Relationship &relationship) { - auto mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); + auto *mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Path &path) { - auto mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); + auto *mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Date &date) { - auto mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); + auto *mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const LocalTime &local_time) { - auto mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); + auto *mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const LocalDateTime &local_date_time) { - auto mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); + auto *mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Duration &duration) { - auto mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); + auto *mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } @@ -4086,8 +4086,8 @@ inline void Record::Insert(const char *field_name, const Value &value) { inline RecordFactory::RecordFactory(mgp_result *result) : result_(result) {} -inline const Record RecordFactory::NewRecord() const { - auto record = mgp::result_new_record(result_); +inline Record RecordFactory::NewRecord() const { + auto *record = mgp::result_new_record(result_); if (record == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -4107,85 +4107,85 @@ inline void RecordFactory::SetErrorMessage(const char *error_msg) const { inline Result::Result(mgp_func_result *result) : result_(result) {} inline void Result::SetValue(bool value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_bool, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_bool, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(std::int64_t value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_int, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_int, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(double value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_double, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_double, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(std::string_view value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const char *value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const List &list) { - auto mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); + auto *mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Map &map) { - auto mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); + auto *mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Node &node) { - auto mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); + auto *mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Relationship &relationship) { - auto mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); + auto *mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Path &path) { - auto mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); + auto *mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Date &date) { - auto mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); + auto *mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const LocalTime &local_time) { - auto mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); + auto *mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const LocalDateTime &local_date_time) { - auto mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); + auto *mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Duration &duration) { - auto mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); + auto *mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } @@ -4222,7 +4222,7 @@ inline Parameter::Parameter(std::string_view name, Type type, const char *defaul : name(name), type_(type), optional(true), default_value(Value(default_value)) {} inline Parameter::Parameter(std::string_view name, Type type, Value default_value) - : name(name), type_(type), optional(true), default_value(default_value) {} + : name(name), type_(type), optional(true), default_value(std::move(default_value)) {} inline Parameter::Parameter(std::string_view name, std::pair list_type) : name(name), type_(list_type.first), list_item_type_(list_type.second) {} @@ -4232,7 +4232,7 @@ inline Parameter::Parameter(std::string_view name, std::pair list_ty type_(list_type.first), list_item_type_(list_type.second), optional(true), - default_value(default_value) {} + default_value(std::move(default_value)) {} inline mgp_type *Parameter::GetMGPType() const { if (type_ == Type::List) { diff --git a/query_modules/schema.cpp b/query_modules/schema.cpp index d5a657e98..1b3035bab 100644 --- a/query_modules/schema.cpp +++ b/query_modules/schema.cpp @@ -111,7 +111,6 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; - ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); @@ -570,7 +569,6 @@ void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *resul extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { try { mgp::MemoryDispatcherGuard guard{memory}; - ; AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {}, {mgp::Return(Schema::kReturnNodeType, mgp::Type::String), diff --git a/src/audit/log.cpp b/src/audit/log.cpp index 635898a16..4222ebb28 100644 --- a/src/audit/log.cpp +++ b/src/audit/log.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "storage/v2/temporal.hpp" #include "utils/logging.hpp" @@ -87,8 +88,8 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) { return ret; } -Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis) - : storage_directory_(storage_directory), +Log::Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis) + : storage_directory_(std::move(storage_directory)), buffer_size_(buffer_size), buffer_flush_interval_millis_(buffer_flush_interval_millis), started_(false) {} diff --git a/src/audit/log.hpp b/src/audit/log.hpp index 8def3ede5..accf41fe8 100644 --- a/src/audit/log.hpp +++ b/src/audit/log.hpp @@ -36,7 +36,7 @@ class Log { }; public: - Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis); + Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis); ~Log(); diff --git a/src/auth/models.cpp b/src/auth/models.cpp index 263131df3..5415dc08d 100644 --- a/src/auth/models.cpp +++ b/src/auth/models.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -560,20 +561,20 @@ Databases Databases::Deserialize(const nlohmann::json &data) { } #endif -User::User() {} +User::User() = default; User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {} -User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions) - : username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {} +User::User(const std::string &username, std::string password_hash, const Permissions &permissions) + : username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {} #ifdef MG_ENTERPRISE -User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions, +User::User(const std::string &username, std::string password_hash, const Permissions &permissions, FineGrainedAccessHandler fine_grained_access_handler, Databases db_access) : username_(utils::ToLowerCase(username)), - password_hash_(password_hash), + password_hash_(std::move(password_hash)), permissions_(permissions), fine_grained_access_handler_(std::move(fine_grained_access_handler)), - database_access_(db_access) {} + database_access_(std::move(db_access)) {} #endif bool User::CheckPassword(const std::string &password) { diff --git a/src/auth/models.hpp b/src/auth/models.hpp index 33ba28f80..9f66d3119 100644 --- a/src/auth/models.hpp +++ b/src/auth/models.hpp @@ -14,6 +14,7 @@ #include #include +#include #include "dbms/constants.hpp" #include "utils/logging.hpp" @@ -301,8 +302,8 @@ class Databases final { bool Contains(const std::string &db) const; bool GetAllowAll() const { return allow_all_; } - const std::set &GetGrants() const { return grants_dbs_; } - const std::set &GetDenies() const { return denies_dbs_; } + const std::set> &GetGrants() const { return grants_dbs_; } + const std::set> &GetDenies() const { return denies_dbs_; } const std::string &GetDefault() const; nlohmann::json Serialize() const; @@ -310,14 +311,17 @@ class Databases final { static Databases Deserialize(const nlohmann::json &data); private: - Databases(bool allow_all, std::set grant, std::set deny, - const std::string &default_db = dbms::kDefaultDB) - : grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {} + Databases(bool allow_all, std::set> grant, std::set> deny, + std::string default_db = dbms::kDefaultDB) + : grants_dbs_(std::move(grant)), + denies_dbs_(std::move(deny)), + allow_all_(allow_all), + default_db_(std::move(default_db)) {} - std::set grants_dbs_; //!< set of databases with granted access - std::set denies_dbs_; //!< set of databases with denied access - bool allow_all_; //!< flag to allow access to everything (denied overrides this) - std::string default_db_; //!< user's default database + std::set> grants_dbs_; //!< set of databases with granted access + std::set> denies_dbs_; //!< set of databases with denied access + bool allow_all_; //!< flag to allow access to everything (denied overrides this) + std::string default_db_; //!< user's default database }; #endif @@ -327,9 +331,9 @@ class User final { User(); explicit User(const std::string &username); - User(const std::string &username, const std::string &password_hash, const Permissions &permissions); + User(const std::string &username, std::string password_hash, const Permissions &permissions); #ifdef MG_ENTERPRISE - User(const std::string &username, const std::string &password_hash, const Permissions &permissions, + User(const std::string &username, std::string password_hash, const Permissions &permissions, FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {}); #endif User(const User &) = default; diff --git a/src/communication/bolt/client.hpp b/src/communication/bolt/client.hpp index 35be997d0..008fca907 100644 --- a/src/communication/bolt/client.hpp +++ b/src/communication/bolt/client.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "communication/bolt/v1/codes.hpp" @@ -34,8 +35,8 @@ class FailureResponseException : public utils::BasicException { explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {} - FailureResponseException(const std::string &code, const std::string &message) - : utils::BasicException{message}, code_{code} {} + FailureResponseException(std::string code, const std::string &message) + : utils::BasicException{message}, code_{std::move(code)} {} const std::string &code() const { return code_; } SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException) diff --git a/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp b/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp index 0d096352b..e82019d09 100644 --- a/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp +++ b/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -51,7 +51,7 @@ enum class ChunkState : uint8_t { template class ChunkedDecoderBuffer { public: - ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); } + explicit ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); } /** * Reads data from the internal buffer. diff --git a/src/communication/bolt/v1/decoder/decoder.hpp b/src/communication/bolt/v1/decoder/decoder.hpp index 593f514f5..8a8821708 100644 --- a/src/communication/bolt/v1/decoder/decoder.hpp +++ b/src/communication/bolt/v1/decoder/decoder.hpp @@ -401,11 +401,11 @@ class Decoder { } auto &labels = dv.ValueList(); vertex.labels.reserve(labels.size()); - for (size_t i = 0; i < labels.size(); ++i) { - if (labels[i].type() != Value::Type::String) { + for (auto &label : labels) { + if (label.type() != Value::Type::String) { return false; } - vertex.labels.emplace_back(std::move(labels[i].ValueString())); + vertex.labels.emplace_back(std::move(label.ValueString())); } // read properties diff --git a/src/communication/bolt/v1/encoder/base_encoder.hpp b/src/communication/bolt/v1/encoder/base_encoder.hpp index ee34ec085..8ce802b82 100644 --- a/src/communication/bolt/v1/encoder/base_encoder.hpp +++ b/src/communication/bolt/v1/encoder/base_encoder.hpp @@ -111,12 +111,12 @@ class BaseEncoder { void WriteList(const std::vector &value) { WriteTypeSize(value.size(), MarkerList); - for (auto &x : value) WriteValue(x); + for (const auto &x : value) WriteValue(x); } void WriteMap(const std::map &value) { WriteTypeSize(value.size(), MarkerMap); - for (auto &x : value) { + for (const auto &x : value) { WriteString(x.first); WriteValue(x.second); } @@ -205,11 +205,11 @@ class BaseEncoder { WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3); WriteRAW(utils::UnderlyingCast(Signature::Path)); WriteTypeSize(path.vertices.size(), MarkerList); - for (auto &v : path.vertices) WriteVertex(v); + for (const auto &v : path.vertices) WriteVertex(v); WriteTypeSize(path.edges.size(), MarkerList); - for (auto &e : path.edges) WriteEdge(e); + for (const auto &e : path.edges) WriteEdge(e); WriteTypeSize(path.indices.size(), MarkerList); - for (auto &i : path.indices) WriteInt(i); + for (const auto &i : path.indices) WriteInt(i); } void WriteDate(const utils::Date &date) { diff --git a/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp b/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp index 6751ce48b..9e481d0f7 100644 --- a/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp +++ b/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -48,7 +48,7 @@ namespace memgraph::communication::bolt { template class ChunkedEncoderBuffer { public: - ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {} + explicit ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {} /** * Writes n values into the buffer. If n is bigger than whole chunk size diff --git a/src/communication/bolt/v1/encoder/client_encoder.hpp b/src/communication/bolt/v1/encoder/client_encoder.hpp index 49f3f0c16..34a5a9b50 100644 --- a/src/communication/bolt/v1/encoder/client_encoder.hpp +++ b/src/communication/bolt/v1/encoder/client_encoder.hpp @@ -39,7 +39,7 @@ class ClientEncoder : private BaseEncoder { using BaseEncoder::buffer_; public: - ClientEncoder(Buffer &buffer) : BaseEncoder(buffer) {} + explicit ClientEncoder(Buffer &buffer) : BaseEncoder(buffer) {} using BaseEncoder::UpdateVersion; diff --git a/src/communication/bolt/v1/encoder/encoder.hpp b/src/communication/bolt/v1/encoder/encoder.hpp index bc3203d90..028c4bf37 100644 --- a/src/communication/bolt/v1/encoder/encoder.hpp +++ b/src/communication/bolt/v1/encoder/encoder.hpp @@ -32,7 +32,7 @@ class Encoder : private BaseEncoder { using BaseEncoder::buffer_; public: - Encoder(Buffer &buffer) : BaseEncoder(buffer) {} + explicit Encoder(Buffer &buffer) : BaseEncoder(buffer) {} using BaseEncoder::UpdateVersion; diff --git a/src/communication/bolt/v1/states/handlers.hpp b/src/communication/bolt/v1/states/handlers.hpp index 26c995719..f873f0e6e 100644 --- a/src/communication/bolt/v1/states/handlers.hpp +++ b/src/communication/bolt/v1/states/handlers.hpp @@ -93,7 +93,7 @@ State HandlePullDiscard(TSession &session, std::optional n, std::optional AuthenticateUser(TSession &session, Value &metadata) { std::string username; std::string password; if (data["scheme"].ValueString() == "basic") { - if (!data.count("principal")) { // Special case principal = "" + if (!data.contains("principal")) { // Special case principal = "" spdlog::warn("The client didn't supply the principal field! Trying with \"\"..."); data["principal"] = ""; } - if (!data.count("credentials")) { // Special case credentials = "" + if (!data.contains("credentials")) { // Special case credentials = "" spdlog::warn("The client didn't supply the credentials field! Trying with \"\"..."); data["credentials"] = ""; } @@ -118,7 +118,7 @@ std::optional GetMetadataV4(TSession &session, const Marker marker) { } auto &data = metadata.ValueMap(); - if (!data.count("user_agent")) { + if (!data.contains("user_agent")) { spdlog::warn("The client didn't supply the user agent!"); return std::nullopt; } @@ -142,7 +142,7 @@ std::optional GetInitDataV5(TSession &session, const Marker marker) { } const auto &data = metadata.ValueMap(); - if (!data.count("user_agent")) { + if (!data.contains("user_agent")) { spdlog::warn("The client didn't supply the user agent!"); return std::nullopt; } diff --git a/src/communication/bolt/v1/value.hpp b/src/communication/bolt/v1/value.hpp index f17e4e2a6..9f189b1f7 100644 --- a/src/communication/bolt/v1/value.hpp +++ b/src/communication/bolt/v1/value.hpp @@ -91,7 +91,7 @@ struct UnboundedEdge { * The decoder writes data into this structure. */ struct Path { - Path() {} + Path() = default; Path(const std::vector &vertices, const std::vector &edges) { // Helper function. Looks for the given element in the collection. If found, diff --git a/src/communication/client.hpp b/src/communication/client.hpp index c0f7cfad8..64507a321 100644 --- a/src/communication/client.hpp +++ b/src/communication/client.hpp @@ -132,7 +132,7 @@ class Client final { */ class ClientInputStream final { public: - ClientInputStream(Client &client); + explicit ClientInputStream(Client &client); ClientInputStream(const ClientInputStream &) = delete; ClientInputStream(ClientInputStream &&) = delete; @@ -156,7 +156,7 @@ class ClientInputStream final { */ class ClientOutputStream final { public: - ClientOutputStream(Client &client); + explicit ClientOutputStream(Client &client); ClientOutputStream(const ClientOutputStream &) = delete; ClientOutputStream(ClientOutputStream &&) = delete; diff --git a/src/communication/context.cpp b/src/communication/context.cpp index 53cb4586b..18794a4cd 100644 --- a/src/communication/context.cpp +++ b/src/communication/context.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -34,7 +34,7 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) { } ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) { - if (key_file != "" && cert_file != "") { + if (!key_file.empty() && !cert_file.empty()) { MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1, "Couldn't load client certificate from file: {}", cert_file); MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1, @@ -124,7 +124,7 @@ ServerContext &ServerContext::operator=(ServerContext &&other) noexcept { return *this; } -ServerContext::~ServerContext() {} +ServerContext::~ServerContext() = default; SSL_CTX *ServerContext::context() { MG_ASSERT(ctx_); diff --git a/src/communication/helpers.cpp b/src/communication/helpers.cpp index 0f91c0ff6..2374e110a 100644 --- a/src/communication/helpers.cpp +++ b/src/communication/helpers.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -15,7 +15,7 @@ namespace memgraph::communication { -const std::string SslGetLastError() { +std::string SslGetLastError() { char buff[2048]; auto err = ERR_get_error(); ERR_error_string_n(err, buff, sizeof(buff)); diff --git a/src/communication/helpers.hpp b/src/communication/helpers.hpp index b82b68a38..f50697162 100644 --- a/src/communication/helpers.hpp +++ b/src/communication/helpers.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -18,6 +18,6 @@ namespace memgraph::communication { /** * This function reads and returns a string describing the last OpenSSL error. */ -const std::string SslGetLastError(); +std::string SslGetLastError(); } // namespace memgraph::communication diff --git a/src/communication/http/listener.hpp b/src/communication/http/listener.hpp index 029bf5ca1..fac4cfaf3 100644 --- a/src/communication/http/listener.hpp +++ b/src/communication/http/listener.hpp @@ -38,7 +38,7 @@ class Listener final : public std::enable_shared_from_this static std::shared_ptr Create(Args &&...args) { diff --git a/src/communication/listener.hpp b/src/communication/listener.hpp index cbb6c0b2f..066d3c53c 100644 --- a/src/communication/listener.hpp +++ b/src/communication/listener.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -51,13 +52,13 @@ class Listener final { using SessionHandler = Session; public: - Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name, + Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, std::string service_name, size_t workers_count) : data_(data), alive_(false), context_(context), inactivity_timeout_sec_(inactivity_timeout_sec), - service_name_(service_name), + service_name_(std::move(service_name)), workers_count_(workers_count) {} ~Listener() { diff --git a/src/communication/result_stream_faker.hpp b/src/communication/result_stream_faker.hpp index 1c899b31b..f8786dd43 100644 --- a/src/communication/result_stream_faker.hpp +++ b/src/communication/result_stream_faker.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -80,7 +80,7 @@ class ResultStreamFaker { std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); }); // convert all the results into strings, and track max column width - auto &results_data = results.GetResults(); + const auto &results_data = results.GetResults(); std::vector> result_strings(results_data.size(), std::vector(column_widths.size())); for (int row_ind = 0; row_ind < static_cast(results_data.size()); ++row_ind) { diff --git a/src/communication/session.hpp b/src/communication/session.hpp index e44bf9532..a6c439ced 100644 --- a/src/communication/session.hpp +++ b/src/communication/session.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -51,7 +52,8 @@ using InputStream = Buffer::ReadEnd; */ class OutputStream final { public: - OutputStream(std::function write_function) : write_function_(write_function) {} + explicit OutputStream(std::function write_function) + : write_function_(std::move(write_function)) {} OutputStream(const OutputStream &) = delete; OutputStream(OutputStream &&) = delete; diff --git a/src/communication/v2/listener.hpp b/src/communication/v2/listener.hpp index 82d6fc2cb..1061f8c09 100644 --- a/src/communication/v2/listener.hpp +++ b/src/communication/v2/listener.hpp @@ -47,7 +47,7 @@ class Listener final : public std::enable_shared_from_this static std::shared_ptr Create(Args &&...args) { diff --git a/src/communication/v2/session.hpp b/src/communication/v2/session.hpp index 37e55e112..b54607729 100644 --- a/src/communication/v2/session.hpp +++ b/src/communication/v2/session.hpp @@ -76,7 +76,7 @@ using tcp = boost::asio::ip::tcp; class OutputStream final { public: explicit OutputStream(std::function write_function) - : write_function_(write_function) {} + : write_function_(std::move(write_function)) {} OutputStream(const OutputStream &) = delete; OutputStream(OutputStream &&) = delete; diff --git a/src/communication/websocket/server.hpp b/src/communication/websocket/server.hpp index 0853d3ebc..31a592932 100644 --- a/src/communication/websocket/server.hpp +++ b/src/communication/websocket/server.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -16,6 +16,7 @@ #include #include #include +#include #include "communication/websocket/listener.hpp" #include "io/network/endpoint.hpp" @@ -45,7 +46,7 @@ class Server final { class LoggingSink : public spdlog::sinks::base_sink { public: - explicit LoggingSink(std::weak_ptr listener) : listener_(listener) {} + explicit LoggingSink(std::weak_ptr listener) : listener_(std::move(listener)) {} private: void sink_it_(const spdlog::details::log_msg &msg) override; diff --git a/src/dbms/dbms_handler.hpp b/src/dbms/dbms_handler.hpp index 990420bf8..27ab963d4 100644 --- a/src/dbms/dbms_handler.hpp +++ b/src/dbms/dbms_handler.hpp @@ -542,7 +542,7 @@ class DbmsHandler { DatabaseHandler db_handler_; //!< multi-tenancy storage handler std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory - std::set defunct_dbs_; //!< Databases that are in an unknown state due to various failures + std::set> defunct_dbs_; //!< Databases that are in an unknown state due to various failures #else mutable utils::Gatekeeper db_gatekeeper_; //!< Single databases gatekeeper #endif diff --git a/src/dbms/handler.hpp b/src/dbms/handler.hpp index a7622e6b2..568b2fc7c 100644 --- a/src/dbms/handler.hpp +++ b/src/dbms/handler.hpp @@ -38,7 +38,7 @@ class Handler { * @brief Empty Handler constructor. * */ - Handler() {} + Handler() = default; /** * @brief Generate a new context and corresponding configuration. diff --git a/src/glue/SessionHL.cpp b/src/glue/SessionHL.cpp index a84f44974..cc7910d1c 100644 --- a/src/glue/SessionHL.cpp +++ b/src/glue/SessionHL.cpp @@ -10,6 +10,7 @@ // licenses/APL.txt. #include +#include #include "gflags/gflags.h" #include "audit/log.hpp" @@ -272,7 +273,7 @@ void SessionHL::Configure(const std::map *auth @@ -289,7 +290,7 @@ SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context, audit_log_(audit_log), #endif auth_(auth), - endpoint_(endpoint), + endpoint_(std::move(endpoint)), implicit_db_(dbms::kDefaultDB) { // Metrics update memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions); diff --git a/src/glue/SessionHL.hpp b/src/glue/SessionHL.hpp index cc53ae08a..374d2464e 100644 --- a/src/glue/SessionHL.hpp +++ b/src/glue/SessionHL.hpp @@ -23,7 +23,7 @@ class SessionHL final : public memgraph::communication::bolt::Session { public: SessionHL(memgraph::query::InterpreterContext *interpreter_context, - const memgraph::communication::v2::ServerEndpoint &endpoint, + memgraph::communication::v2::ServerEndpoint endpoint, memgraph::communication::v2::InputStream *input_stream, memgraph::communication::v2::OutputStream *output_stream, memgraph::utils::Synchronized *auth diff --git a/src/glue/communication.cpp b/src/glue/communication.cpp index 5db3e2726..fdf5129f6 100644 --- a/src/glue/communication.cpp +++ b/src/glue/communication.cpp @@ -202,7 +202,7 @@ storage::Result> ToBoltGraph(const query::Graph &gr for (const auto &v : graph.vertices()) { auto maybe_vertex = ToBoltVertex(v, db, view); if (maybe_vertex.HasError()) return maybe_vertex.GetError(); - vertices.emplace_back(Value(std::move(*maybe_vertex))); + vertices.emplace_back(std::move(*maybe_vertex)); } map.emplace("nodes", Value(vertices)); @@ -211,7 +211,7 @@ storage::Result> ToBoltGraph(const query::Graph &gr for (const auto &e : graph.edges()) { auto maybe_edge = ToBoltEdge(e, db, view); if (maybe_edge.HasError()) return maybe_edge.GetError(); - edges.emplace_back(Value(std::move(*maybe_edge))); + edges.emplace_back(std::move(*maybe_edge)); } map.emplace("edges", Value(edges)); diff --git a/src/helpers.hpp b/src/helpers.hpp index 1cf4e0ec0..67c1906e7 100644 --- a/src/helpers.hpp +++ b/src/helpers.hpp @@ -31,7 +31,7 @@ inline void LoadConfig(const std::string &product_name) { std::vector configs = {fs::path("/etc/memgraph/memgraph.conf")}; if (getenv("HOME") != nullptr) configs.emplace_back(fs::path(getenv("HOME")) / fs::path(".memgraph/config")); { - auto memgraph_config = getenv("MEMGRAPH_CONFIG"); + auto *memgraph_config = getenv("MEMGRAPH_CONFIG"); if (memgraph_config != nullptr) { auto path = fs::path(memgraph_config); MG_ASSERT(fs::exists(path), "MEMGRAPH_CONFIG environment variable set to nonexisting path: {}", diff --git a/src/http_handlers/metrics.hpp b/src/http_handlers/metrics.hpp index d4620b774..82131d311 100644 --- a/src/http_handlers/metrics.hpp +++ b/src/http_handlers/metrics.hpp @@ -23,7 +23,6 @@ #include #include #include "storage/v2/storage.hpp" -#include "utils/event_gauge.hpp" #include "utils/event_histogram.hpp" namespace memgraph::http { diff --git a/src/integrations/kafka/consumer.hpp b/src/integrations/kafka/consumer.hpp index 34e8b1bdb..5f698a4d1 100644 --- a/src/integrations/kafka/consumer.hpp +++ b/src/integrations/kafka/consumer.hpp @@ -171,10 +171,10 @@ class Consumer final : public RdKafka::EventCb { class ConsumerRebalanceCb : public RdKafka::RebalanceCb { public: - ConsumerRebalanceCb(std::string consumer_name); + explicit ConsumerRebalanceCb(std::string consumer_name); void rebalance_cb(RdKafka::KafkaConsumer *consumer, RdKafka::ErrorCode err, - std::vector &partitions) override final; + std::vector &partitions) final; void set_offset(int64_t offset); diff --git a/src/integrations/pulsar/consumer.cpp b/src/integrations/pulsar/consumer.cpp index b3a65050b..f004cf6dc 100644 --- a/src/integrations/pulsar/consumer.cpp +++ b/src/integrations/pulsar/consumer.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -67,7 +67,7 @@ utils::BasicResult> GetBatch(TConsumer &consum return std::move(batch); case pulsar_client::Result::ResultOk: if (message.getMessageId() != last_message_id) { - batch.emplace_back(Message{std::move(message)}); + batch.emplace_back(std::move(message)); } break; default: diff --git a/src/io/network/endpoint.hpp b/src/io/network/endpoint.hpp index e10c80657..281be2162 100644 --- a/src/io/network/endpoint.hpp +++ b/src/io/network/endpoint.hpp @@ -48,8 +48,8 @@ struct Endpoint { uint16_t port{0}; IpFamily family{IpFamily::NONE}; - static std::optional> ParseSocketOrAddress( - const std::string &address, const std::optional default_port); + static std::optional> ParseSocketOrAddress(const std::string &address, + std::optional default_port); /** * Tries to parse the given string as either a socket address or ip address. @@ -61,8 +61,8 @@ struct Endpoint { * it into an ip address and a port number; even if a default port is given, * it won't be used, as we expect that it is given in the address string. */ - static std::optional> ParseSocketOrIpAddress( - const std::string &address, const std::optional default_port); + static std::optional> ParseSocketOrIpAddress(const std::string &address, + std::optional default_port); /** * Tries to parse given string as either socket address or hostname. @@ -72,7 +72,7 @@ struct Endpoint { * After we parse hostname and port we try to resolve the hostname into an ip_address. */ static std::optional> ParseHostname(const std::string &address, - const std::optional default_port); + std::optional default_port); static IpFamily GetIpFamily(const std::string &address); diff --git a/src/io/network/epoll.hpp b/src/io/network/epoll.hpp index 09c485eb5..2f0d0dfd1 100644 --- a/src/io/network/epoll.hpp +++ b/src/io/network/epoll.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -32,7 +32,7 @@ class Epoll { public: using Event = struct epoll_event; - Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) { + explicit Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) { // epoll_create1 returns an error if there is a logical error in our code // (for example invalid flags) or if there is irrecoverable error. In both // cases it is best to terminate. diff --git a/src/io/network/socket.hpp b/src/io/network/socket.hpp index 22e787611..c2b6354af 100644 --- a/src/io/network/socket.hpp +++ b/src/io/network/socket.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "io/network/endpoint.hpp" @@ -201,7 +202,7 @@ class Socket { bool WaitForReadyWrite(); private: - Socket(int fd, const Endpoint &endpoint) : socket_(fd), endpoint_(endpoint) {} + Socket(int fd, Endpoint endpoint) : socket_(fd), endpoint_(std::move(endpoint)) {} int socket_ = -1; Endpoint endpoint_; diff --git a/src/kvstore/kvstore.cpp b/src/kvstore/kvstore.cpp index 383324238..877d6f9bd 100644 --- a/src/kvstore/kvstore.cpp +++ b/src/kvstore/kvstore.cpp @@ -128,7 +128,7 @@ KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, b KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); } -KVStore::iterator::~iterator() {} +KVStore::iterator::~iterator() = default; KVStore::iterator &KVStore::iterator::operator=(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); diff --git a/src/query/common.hpp b/src/query/common.hpp index 8f1b0a94c..6f45760fe 100644 --- a/src/query/common.hpp +++ b/src/query/common.hpp @@ -41,7 +41,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b); /// the define how respective elements compare. class TypedValueVectorCompare final { public: - TypedValueVectorCompare() {} + TypedValueVectorCompare() = default; explicit TypedValueVectorCompare(const std::vector &ordering) : ordering_(ordering) {} template @@ -147,8 +147,8 @@ concept AccessorWithUpdateProperties = requires(T accessor, /// /// @throw QueryRuntimeException if value cannot be set as a property value template -auto UpdatePropertiesChecked(T *record, std::map &properties) -> - typename std::remove_referenceUpdateProperties(properties).GetValue())>::type { +auto UpdatePropertiesChecked(T *record, std::map &properties) + -> std::remove_reference_tUpdateProperties(properties).GetValue())> { try { auto maybe_values = record->UpdateProperties(properties); if (maybe_values.HasError()) { diff --git a/src/query/cypher_query_interpreter.hpp b/src/query/cypher_query_interpreter.hpp index f33fa61e2..f1e9113a3 100644 --- a/src/query/cypher_query_interpreter.hpp +++ b/src/query/cypher_query_interpreter.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "query/config.hpp" #include "query/frontend/semantic/required_privileges.hpp" #include "query/frontend/semantic/symbol_generator.hpp" @@ -98,8 +100,8 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map root, double cost, AstStorage storage, - const SymbolTable &symbol_table) - : root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {} + SymbolTable symbol_table) + : root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(std::move(symbol_table)) {} const plan::LogicalOperator &GetRoot() const override { return *root_; } double GetCost() const override { return cost_; } diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index 75ec1e9ae..f616dc5a2 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -40,7 +40,6 @@ class EdgeAccessor final { public: storage::EdgeAccessor impl_; - public: explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {} bool IsVisible(storage::View view) const { return impl_.IsVisible(view); } @@ -108,7 +107,6 @@ class VertexAccessor final { static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); } - public: explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {} bool IsVisible(storage::View view) const { return impl_.IsVisible(view); } diff --git a/src/query/dump.cpp b/src/query/dump.cpp index dd99f0f63..a1421cbf9 100644 --- a/src/query/dump.cpp +++ b/src/query/dump.cpp @@ -159,7 +159,7 @@ void DumpProperties(std::ostream *os, query::DbAccessor *dba, *os << "{"; if (property_id) { *os << kInternalPropertyId << ": " << *property_id; - if (store.size() > 0) *os << ", "; + if (!store.empty()) *os << ", "; } utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) { os << EscapeName(dba->PropertyToName(kv.first)) << ": "; @@ -228,7 +228,7 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba, const query::EdgeAccesso throw query::QueryRuntimeException("Unexpected error when getting properties."); } } - if (maybe_props->size() > 0) { + if (!maybe_props->empty()) { *os << " "; DumpProperties(os, dba, *maybe_props); } diff --git a/src/query/frame_change.hpp b/src/query/frame_change.hpp index 1d9ebc70c..32fe1f36e 100644 --- a/src/query/frame_change.hpp +++ b/src/query/frame_change.hpp @@ -47,9 +47,9 @@ struct CachedValue { if (!maybe_list.IsList()) { return false; } - auto &list = maybe_list.ValueList(); + const auto &list = maybe_list.ValueList(); TypedValue::Hash hash{}; - for (auto &element : list) { + for (const auto &element : list) { const auto key = hash(element); auto &vector_values = cache_[key]; if (!IsValueInVec(vector_values, element)) { diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index 6b7fdb2c6..dc11c3887 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -23,9 +23,7 @@ #include "storage/v2/property_value.hpp" #include "utils/typeinfo.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { struct LabelIx { static const utils::TypeInfo kType; @@ -62,8 +60,8 @@ inline bool operator!=(const PropertyIx &a, const PropertyIx &b) { return !(a == inline bool operator==(const EdgeTypeIx &a, const EdgeTypeIx &b) { return a.ix == b.ix && a.name == b.name; } inline bool operator!=(const EdgeTypeIx &a, const EdgeTypeIx &b) { return !(a == b); } -} // namespace query -} // namespace memgraph +} // namespace memgraph::query + namespace std { template <> @@ -83,9 +81,7 @@ struct hash { } // namespace std -namespace memgraph { - -namespace query { +namespace memgraph::query { class Tree; @@ -3577,5 +3573,4 @@ class ShowDatabasesQuery : public memgraph::query::Query { } }; -} // namespace query -} // namespace memgraph +} // namespace memgraph::query diff --git a/src/query/frontend/ast/pretty_print.cpp b/src/query/frontend/ast/pretty_print.cpp index 8275f1cb5..ef45afd7d 100644 --- a/src/query/frontend/ast/pretty_print.cpp +++ b/src/query/frontend/ast/pretty_print.cpp @@ -105,7 +105,7 @@ void PrintObject(std::ostream *out, const std::map &map); template void PrintObject(std::ostream *out, const T &arg) { - static_assert(!std::is_convertible::value, + static_assert(!std::is_convertible_v, "This overload shouldn't be called with pointers convertible " "to Expression *. This means your other PrintObject overloads aren't " "being called for certain AST nodes when they should (or perhaps such " diff --git a/src/query/frontend/opencypher/parser.hpp b/src/query/frontend/opencypher/parser.hpp index 3a17d27a8..f25d6e5e9 100644 --- a/src/query/frontend/opencypher/parser.hpp +++ b/src/query/frontend/opencypher/parser.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -31,7 +31,7 @@ class Parser { * @param query incoming query that has to be compiled into query plan * the first step is to generate AST */ - Parser(const std::string query) : query_(std::move(query)) { + explicit Parser(const std::string query) : query_(std::move(query)) { parser_.removeErrorListeners(); parser_.addErrorListener(&error_listener_); tree_ = parser_.cypher(); diff --git a/src/query/frontend/semantic/symbol.hpp b/src/query/frontend/semantic/symbol.hpp index 5381cb48d..77557b6fe 100644 --- a/src/query/frontend/semantic/symbol.hpp +++ b/src/query/frontend/semantic/symbol.hpp @@ -12,12 +12,11 @@ #pragma once #include +#include #include "utils/typeinfo.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { class Symbol { public: @@ -34,9 +33,13 @@ class Symbol { return enum_string[static_cast(type)]; } - Symbol() {} - Symbol(const std::string &name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1) - : name_(name), position_(position), user_declared_(user_declared), type_(type), token_position_(token_position) {} + Symbol() = default; + Symbol(std::string name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1) + : name_(std::move(name)), + position_(position), + user_declared_(user_declared), + type_(type), + token_position_(token_position) {} bool operator==(const Symbol &other) const { return position_ == other.position_ && name_ == other.name_ && type_ == other.type_; @@ -57,8 +60,8 @@ class Symbol { int64_t token_position_{-1}; }; -} // namespace query -} // namespace memgraph +} // namespace memgraph::query + namespace std { template <> diff --git a/src/query/frontend/semantic/symbol_generator.hpp b/src/query/frontend/semantic/symbol_generator.hpp index c69d8729e..207bbddbd 100644 --- a/src/query/frontend/semantic/symbol_generator.hpp +++ b/src/query/frontend/semantic/symbol_generator.hpp @@ -183,7 +183,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor { /// If property lookup for one symbol is visited more times, it is better to fetch all properties class PropertyLookupEvaluationModeVisitor : public ExpressionVisitor { public: - explicit PropertyLookupEvaluationModeVisitor() {} + explicit PropertyLookupEvaluationModeVisitor() = default; using ExpressionVisitor::Visit; diff --git a/src/query/frontend/semantic/symbol_table.hpp b/src/query/frontend/semantic/symbol_table.hpp index 23b4965c5..0b521356c 100644 --- a/src/query/frontend/semantic/symbol_table.hpp +++ b/src/query/frontend/semantic/symbol_table.hpp @@ -22,7 +22,7 @@ namespace memgraph::query { class SymbolTable final { public: - SymbolTable() {} + SymbolTable() = default; const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY, int32_t token_position = -1) { MG_ASSERT(table_.size() <= std::numeric_limits::max(), diff --git a/src/query/frontend/stripped.cpp b/src/query/frontend/stripped.cpp index 56d29032c..9740cd463 100644 --- a/src/query/frontend/stripped.cpp +++ b/src/query/frontend/stripped.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "query/exceptions.hpp" @@ -32,7 +33,7 @@ namespace memgraph::query::frontend { using namespace lexer_constants; -StrippedQuery::StrippedQuery(const std::string &query) : original_(query) { +StrippedQuery::StrippedQuery(std::string query) : original_(std::move(query)) { enum class Token { UNMATCHED, KEYWORD, // Including true, false and null. @@ -255,29 +256,29 @@ std::string GetFirstUtf8Symbol(const char *_s) { // According to // https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe // this checks if casting from const char * to uint8_t is undefined behaviour. - static_assert(std::is_same::value, + static_assert(std::is_same_v, "This library requires std::uint8_t to be implemented as " "unsigned char."); const uint8_t *s = reinterpret_cast(_s); if ((*s >> 7) == 0x00) return std::string(_s, _s + 1); if ((*s >> 5) == 0x06) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 2); } if ((*s >> 4) == 0x0e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 3); } if ((*s >> 3) == 0x1e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s3 = s + 3; + const auto *s3 = s + 3; if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 4); } @@ -286,29 +287,29 @@ std::string GetFirstUtf8Symbol(const char *_s) { // Return codepoint of first utf8 symbol and its encoded length. std::pair GetFirstUtf8SymbolCodepoint(const char *_s) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "This library requires std::uint8_t to be implemented as " "unsigned char."); const uint8_t *s = reinterpret_cast(_s); if ((*s >> 7) == 0x00) return {*s & 0x7f, 1}; if ((*s >> 5) == 0x06) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2}; } if ((*s >> 4) == 0x0e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3}; } if ((*s >> 3) == 0x1e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s3 = s + 3; + const auto *s3 = s + 3; if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4}; } @@ -336,7 +337,7 @@ int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(o int StrippedQuery::MatchString(int start) const { if (original_[start] != '"' && original_[start] != '\'') return 0; char start_char = original_[start]; - for (auto *p = original_.data() + start + 1; *p; ++p) { + for (const auto *p = original_.data() + start + 1; *p; ++p) { if (*p == start_char) return p - (original_.data() + start) + 1; if (*p == '\\') { ++p; @@ -346,7 +347,7 @@ int StrippedQuery::MatchString(int start) const { continue; } else if (*p == 'U' || *p == 'u') { int cnt = 0; - auto *r = p + 1; + const auto *r = p + 1; while (isxdigit(*r) && cnt < 8) { ++cnt; ++r; diff --git a/src/query/frontend/stripped.hpp b/src/query/frontend/stripped.hpp index 006fb1dd6..a20e28474 100644 --- a/src/query/frontend/stripped.hpp +++ b/src/query/frontend/stripped.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -40,7 +40,7 @@ class StrippedQuery { * * @param query Input query. */ - explicit StrippedQuery(const std::string &query); + explicit StrippedQuery(std::string query); /** * Copy constructor is deleted because we don't want to make unnecessary diff --git a/src/query/frontend/stripped_lexer_constants.hpp b/src/query/frontend/stripped_lexer_constants.hpp index 9ad95e6f5..21a14ae83 100644 --- a/src/query/frontend/stripped_lexer_constants.hpp +++ b/src/query/frontend/stripped_lexer_constants.hpp @@ -17,8 +17,7 @@ #include #include -namespace memgraph::query { -namespace lexer_constants { +namespace memgraph::query::lexer_constants { namespace trie { @@ -33,7 +32,7 @@ inline int Noop(int x) { return x; } class Trie { public: - Trie() {} + Trie() = default; Trie(std::initializer_list l) { for (const auto &s : l) { Insert(s); @@ -2934,5 +2933,4 @@ const trie::Trie kSpecialTokens = {";", "\xEF\xB9\x98", // u8"\ufe58" "\xEF\xB9\xA3", // u8"\ufe63" "\xEF\xBC\x8D"}; // u8"\uff0d" -} // namespace lexer_constants -} // namespace memgraph::query +} // namespace memgraph::query::lexer_constants diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index 333a7b1fa..916082bb2 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -825,8 +825,8 @@ class ExpressionEvaluator : public ExpressionVisitor { throw QueryRuntimeException("'coalesce' requires at least one argument."); } - for (int64_t i = 0; i < exprs.size(); ++i) { - TypedValue val(exprs[i]->Accept(*this), ctx_->memory); + for (auto &expr : exprs) { + TypedValue val(expr->Accept(*this), ctx_->memory); if (!val.IsNull()) { return val; } diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 5aad0ff07..354f13dc3 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -797,34 +797,33 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & std::vector typed_replica; typed_replica.reserve(replica_nfields); - typed_replica.emplace_back(TypedValue(replica.name)); - typed_replica.emplace_back(TypedValue(replica.socket_address)); + typed_replica.emplace_back(replica.name); + typed_replica.emplace_back(replica.socket_address); switch (replica.sync_mode) { case ReplicationQuery::SyncMode::SYNC: - typed_replica.emplace_back(TypedValue("sync")); + typed_replica.emplace_back("sync"); break; case ReplicationQuery::SyncMode::ASYNC: - typed_replica.emplace_back(TypedValue("async")); + typed_replica.emplace_back("async"); break; } - typed_replica.emplace_back(TypedValue(static_cast(replica.current_timestamp_of_replica))); - typed_replica.emplace_back( - TypedValue(static_cast(replica.current_number_of_timestamp_behind_master))); + typed_replica.emplace_back(static_cast(replica.current_timestamp_of_replica)); + typed_replica.emplace_back(static_cast(replica.current_number_of_timestamp_behind_master)); switch (replica.state) { case ReplicationQuery::ReplicaState::READY: - typed_replica.emplace_back(TypedValue("ready")); + typed_replica.emplace_back("ready"); break; case ReplicationQuery::ReplicaState::REPLICATING: - typed_replica.emplace_back(TypedValue("replicating")); + typed_replica.emplace_back("replicating"); break; case ReplicationQuery::ReplicaState::RECOVERY: - typed_replica.emplace_back(TypedValue("recovery")); + typed_replica.emplace_back("recovery"); break; case ReplicationQuery::ReplicaState::INVALID: - typed_replica.emplace_back(TypedValue("invalid")); + typed_replica.emplace_back("invalid"); break; } @@ -1962,11 +1961,11 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat result.reserve(kComputeStatisticsNumResults); result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first)); - result.emplace_back(TypedValue()); + result.emplace_back(); result.emplace_back(static_cast(stat_entry.second.count)); - result.emplace_back(TypedValue()); - result.emplace_back(TypedValue()); - result.emplace_back(TypedValue()); + result.emplace_back(); + result.emplace_back(); + result.emplace_back(); result.emplace_back(stat_entry.second.avg_degree); results.push_back(std::move(result)); }); @@ -2883,7 +2882,7 @@ auto ShowTransactions(const std::unordered_set &interpreters, con metadata_tv.emplace(md.first, TypedValue(md.second)); } } - results.back().push_back(TypedValue(metadata_tv)); + results.back().emplace_back(metadata_tv); } } return results; diff --git a/src/query/interpreter.hpp b/src/query/interpreter.hpp index 66231059d..5cb73cb07 100644 --- a/src/query/interpreter.hpp +++ b/src/query/interpreter.hpp @@ -174,7 +174,7 @@ struct CurrentDB { class Interpreter final { public: - Interpreter(InterpreterContext *interpreter_context); + explicit Interpreter(InterpreterContext *interpreter_context); Interpreter(InterpreterContext *interpreter_context, memgraph::dbms::DatabaseAccess db); Interpreter(const Interpreter &) = delete; Interpreter &operator=(const Interpreter &) = delete; diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index b68810ad7..1c8d021c7 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -207,8 +207,8 @@ void Once::OnceCursor::Shutdown() {} void Once::OnceCursor::Reset() { did_pull_ = false; } -CreateNode::CreateNode(const std::shared_ptr &input, const NodeCreationInfo &node_info) - : input_(input ? input : std::make_shared()), node_info_(node_info) {} +CreateNode::CreateNode(const std::shared_ptr &input, NodeCreationInfo node_info) + : input_(input ? input : std::make_shared()), node_info_(std::move(node_info)) {} // Creates a vertex on this GraphDb. Returns a reference to vertex placed on the // frame. @@ -298,12 +298,12 @@ void CreateNode::CreateNodeCursor::Shutdown() { input_cursor_->Shutdown(); } void CreateNode::CreateNodeCursor::Reset() { input_cursor_->Reset(); } -CreateExpand::CreateExpand(const NodeCreationInfo &node_info, const EdgeCreationInfo &edge_info, +CreateExpand::CreateExpand(NodeCreationInfo node_info, EdgeCreationInfo edge_info, const std::shared_ptr &input, Symbol input_symbol, bool existing_node) - : node_info_(node_info), - edge_info_(edge_info), + : node_info_(std::move(node_info)), + edge_info_(std::move(edge_info)), input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), existing_node_(existing_node) {} ACCEPT_WITH_INPUT(CreateExpand) @@ -447,7 +447,7 @@ class ScanAllCursor : public Cursor { explicit ScanAllCursor(const ScanAll &self, Symbol output_symbol, UniqueCursorPtr input_cursor, storage::View view, TVerticesFun get_vertices, const char *op_name) : self_(self), - output_symbol_(output_symbol), + output_symbol_(std::move(output_symbol)), input_cursor_(std::move(input_cursor)), view_(view), get_vertices_(std::move(get_vertices)), @@ -518,7 +518,7 @@ class ScanAllCursor : public Cursor { }; ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::View view) - : input_(input ? input : std::make_shared()), output_symbol_(output_symbol), view_(view) {} + : input_(input ? input : std::make_shared()), output_symbol_(std::move(output_symbol)), view_(view) {} ACCEPT_WITH_INPUT(ScanAll) @@ -561,13 +561,13 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const { ScanAllByLabelPropertyRange::ScanAllByLabelPropertyRange(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, std::optional lower_bound, std::optional upper_bound, storage::View view) : ScanAll(input, output_symbol, view), label_(label), property_(property), - property_name_(property_name), + property_name_(std::move(property_name)), lower_bound_(lower_bound), upper_bound_(upper_bound) { MG_ASSERT(lower_bound_ || upper_bound_, "Only one bound can be left out"); @@ -623,12 +623,12 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, Expression *expression, storage::View view) : ScanAll(input, output_symbol, view), label_(label), property_(property), - property_name_(property_name), + property_name_(std::move(property_name)), expression_(expression) { DMG_ASSERT(expression, "Expression is not optional."); } @@ -655,8 +655,11 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, storage::PropertyId property, - const std::string &property_name, storage::View view) - : ScanAll(input, output_symbol, view), label_(label), property_(property), property_name_(property_name) {} + std::string property_name, storage::View view) + : ScanAll(input, output_symbol, view), + label_(label), + property_(property), + property_name_(std::move(property_name)) {} ACCEPT_WITH_INPUT(ScanAllByLabelProperty) @@ -728,7 +731,7 @@ Expand::Expand(const std::shared_ptr &input, Symbol input_symbo Symbol edge_symbol, EdgeAtom::Direction direction, const std::vector &edge_types, bool existing_node, storage::View view) : input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, view_(view) {} @@ -962,15 +965,15 @@ ExpandVariable::ExpandVariable(const std::shared_ptr &input, Sy ExpansionLambda filter_lambda, std::optional weight_lambda, std::optional total_weight) : input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, type_(type), is_reverse_(is_reverse), lower_bound_(lower_bound), upper_bound_(upper_bound), - filter_lambda_(filter_lambda), - weight_lambda_(weight_lambda), - total_weight_(total_weight) { + filter_lambda_(std::move(filter_lambda)), + weight_lambda_(std::move(weight_lambda)), + total_weight_(std::move(total_weight)) { DMG_ASSERT(type_ == EdgeAtom::Type::DEPTH_FIRST || type_ == EdgeAtom::Type::BREADTH_FIRST || type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS, "ExpandVariable can only be used with breadth first, depth first, " @@ -1758,7 +1761,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { if (found_it != total_cost_.end() && (found_it->second.IsNull() || (found_it->second <= next_weight).ValueBool())) return; - pq_.push({next_weight, depth + 1, vertex, edge}); + pq_.emplace(next_weight, depth + 1, vertex, edge); }; // Populates the priority queue structure with expansions @@ -1810,7 +1813,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { total_cost_.clear(); yielded_vertices_.clear(); - pq_.push({TypedValue(), 0, vertex, std::nullopt}); + pq_.emplace(TypedValue(), 0, vertex, std::nullopt); // We are adding the starting vertex to the set of yielded vertices // because we don't want to yield paths that end with the starting // vertex. @@ -2023,7 +2026,7 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor { } DirectedEdge directed_edge = {edge, direction, next_weight}; - pq_.push({next_weight, depth + 1, next_vertex, directed_edge}); + pq_.emplace(next_weight, depth + 1, next_vertex, directed_edge); }; // Populates the priority queue structure with expansions @@ -2314,8 +2317,8 @@ UniqueCursorPtr ExpandVariable::MakeCursor(utils::MemoryResource *mem) const { class ConstructNamedPathCursor : public Cursor { public: - ConstructNamedPathCursor(const ConstructNamedPath &self, utils::MemoryResource *mem) - : self_(self), input_cursor_(self_.input()->MakeCursor(mem)) {} + ConstructNamedPathCursor(ConstructNamedPath self, utils::MemoryResource *mem) + : self_(std::move(self)), input_cursor_(self_.input()->MakeCursor(mem)) {} bool Pull(Frame &frame, ExecutionContext &context) override { OOMExceptionEnabler oom_exception; @@ -2413,11 +2416,11 @@ Filter::Filter(const std::shared_ptr &input, Filter::Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression, - const Filters &all_filters) + Filters all_filters) : input_(input ? input : std::make_shared()), pattern_filters_(pattern_filters), expression_(expression), - all_filters_(all_filters) {} + all_filters_(std::move(all_filters)) {} bool Filter::Accept(HierarchicalLogicalOperatorVisitor &visitor) { if (visitor.PreVisit(*this)) { @@ -2478,7 +2481,7 @@ void Filter::FilterCursor::Shutdown() { input_cursor_->Shutdown(); } void Filter::FilterCursor::Reset() { input_cursor_->Reset(); } EvaluatePatternFilter::EvaluatePatternFilter(const std::shared_ptr &input, Symbol output_symbol) - : input_(input), output_symbol_(output_symbol) {} + : input_(input), output_symbol_(std::move(output_symbol)) {} ACCEPT_WITH_INPUT(EvaluatePatternFilter); @@ -2801,7 +2804,7 @@ void SetProperty::SetPropertyCursor::Shutdown() { input_cursor_->Shutdown(); } void SetProperty::SetPropertyCursor::Reset() { input_cursor_->Reset(); } SetProperties::SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op) - : input_(input), input_symbol_(input_symbol), rhs_(rhs), op_(op) {} + : input_(input), input_symbol_(std::move(input_symbol)), rhs_(rhs), op_(op) {} ACCEPT_WITH_INPUT(SetProperties) @@ -3000,7 +3003,7 @@ void SetProperties::SetPropertiesCursor::Reset() { input_cursor_->Reset(); } SetLabels::SetLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels) - : input_(input), input_symbol_(input_symbol), labels_(labels) {} + : input_(input), input_symbol_(std::move(input_symbol)), labels_(labels) {} ACCEPT_WITH_INPUT(SetLabels) @@ -3160,7 +3163,7 @@ void RemoveProperty::RemovePropertyCursor::Reset() { input_cursor_->Reset(); } RemoveLabels::RemoveLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels) - : input_(input), input_symbol_(input_symbol), labels_(labels) {} + : input_(input), input_symbol_(std::move(input_symbol)), labels_(labels) {} ACCEPT_WITH_INPUT(RemoveLabels) @@ -3234,7 +3237,7 @@ void RemoveLabels::RemoveLabelsCursor::Reset() { input_cursor_->Reset(); } EdgeUniquenessFilter::EdgeUniquenessFilter(const std::shared_ptr &input, Symbol expand_symbol, const std::vector &previous_symbols) - : input_(input), expand_symbol_(expand_symbol), previous_symbols_(previous_symbols) {} + : input_(input), expand_symbol_(std::move(expand_symbol)), previous_symbols_(previous_symbols) {} ACCEPT_WITH_INPUT(EdgeUniquenessFilter) @@ -4204,7 +4207,7 @@ void Optional::OptionalCursor::Reset() { Unwind::Unwind(const std::shared_ptr &input, Expression *input_expression, Symbol output_symbol) : input_(input ? input : std::make_shared()), input_expression_(input_expression), - output_symbol_(output_symbol) {} + output_symbol_(std::move(output_symbol)) {} ACCEPT_WITH_INPUT(Unwind) @@ -4535,7 +4538,7 @@ WITHOUT_SINGLE_INPUT(OutputTable); class OutputTableCursor : public Cursor { public: - OutputTableCursor(const OutputTable &self) : self_(self) {} + explicit OutputTableCursor(const OutputTable &self) : self_(self) {} bool Pull(Frame &frame, ExecutionContext &context) override { OOMExceptionEnabler oom_exception; @@ -4626,10 +4629,10 @@ CallProcedure::CallProcedure(std::shared_ptr input, std::string std::vector fields, std::vector symbols, Expression *memory_limit, size_t memory_scale, bool is_write, int64_t procedure_id, bool void_procedure) : input_(input ? input : std::make_shared()), - procedure_name_(name), - arguments_(args), - result_fields_(fields), - result_symbols_(symbols), + procedure_name_(std::move(name)), + arguments_(std::move(args)), + result_fields_(std::move(fields)), + result_symbols_(std::move(symbols)), memory_limit_(memory_limit), memory_scale_(memory_scale), is_write_(is_write), @@ -4983,7 +4986,7 @@ LoadCsv::LoadCsv(std::shared_ptr input, Expression *file, bool delimiter_(delimiter), quote_(quote), nullif_(nullif), - row_var_(row_var) { + row_var_(std::move(row_var)) { MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__); } @@ -5197,7 +5200,7 @@ Foreach::Foreach(std::shared_ptr input, std::shared_ptr()), update_clauses_(std::move(updates)), expression_(expr), - loop_variable_symbol_(loop_variable_symbol) {} + loop_variable_symbol_(std::move(loop_variable_symbol)) {} UniqueCursorPtr Foreach::MakeCursor(utils::MemoryResource *mem) const { memgraph::metrics::IncrementCounter(memgraph::metrics::ForeachOperator); diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index 7bb971752..03df07378 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -32,9 +32,7 @@ #include "utils/synchronized.hpp" #include "utils/visitor.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { struct ExecutionContext; class ExpressionEvaluator; @@ -68,7 +66,7 @@ class Cursor { /// Perform cleanup which may throw an exception virtual void Shutdown() = 0; - virtual ~Cursor() {} + virtual ~Cursor() = default; }; /// unique_ptr to Cursor managed with a custom deleter. @@ -172,7 +170,7 @@ class LogicalOperator : public utils::Visitable &input, const NodeCreationInfo &node_info); + CreateNode(const std::shared_ptr &input, NodeCreationInfo node_info); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -445,7 +443,7 @@ class CreateExpand : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - CreateExpand() {} + CreateExpand() = default; /** @brief Construct @c CreateExpand. * @@ -459,8 +457,8 @@ class CreateExpand : public memgraph::query::plan::LogicalOperator { * @param existing_node @c bool indicating whether the @c node_atom refers to * an existing node. If @c false, the operator will also create the node. */ - CreateExpand(const NodeCreationInfo &node_info, const EdgeCreationInfo &edge_info, - const std::shared_ptr &input, Symbol input_symbol, bool existing_node); + CreateExpand(NodeCreationInfo node_info, EdgeCreationInfo edge_info, const std::shared_ptr &input, + Symbol input_symbol, bool existing_node); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -529,7 +527,7 @@ class ScanAll : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAll() {} + ScanAll() = default; ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -571,7 +569,7 @@ class ScanAllByLabel : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabel() {} + ScanAllByLabel() = default; ScanAllByLabel(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -606,7 +604,7 @@ class ScanAllByLabelPropertyRange : public memgraph::query::plan::ScanAll { /** Bound with expression which when evaluated produces the bound value. */ using Bound = utils::Bound; - ScanAllByLabelPropertyRange() {} + ScanAllByLabelPropertyRange() = default; /** * Constructs the operator for given label and property value in range * (inclusive). @@ -622,7 +620,7 @@ class ScanAllByLabelPropertyRange : public memgraph::query::plan::ScanAll { * @param view storage::View used when obtaining vertices. */ ScanAllByLabelPropertyRange(const std::shared_ptr &input, Symbol output_symbol, - storage::LabelId label, storage::PropertyId property, const std::string &property_name, + storage::LabelId label, storage::PropertyId property, std::string property_name, std::optional lower_bound, std::optional upper_bound, storage::View view = storage::View::OLD); @@ -675,7 +673,7 @@ class ScanAllByLabelPropertyValue : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabelPropertyValue() {} + ScanAllByLabelPropertyValue() = default; /** * Constructs the operator for given label and property value. * @@ -687,7 +685,7 @@ class ScanAllByLabelPropertyValue : public memgraph::query::plan::ScanAll { * @param view storage::View used when obtaining vertices. */ ScanAllByLabelPropertyValue(const std::shared_ptr &input, Symbol output_symbol, - storage::LabelId label, storage::PropertyId property, const std::string &property_name, + storage::LabelId label, storage::PropertyId property, std::string property_name, Expression *expression, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -727,9 +725,9 @@ class ScanAllByLabelProperty : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabelProperty() {} + ScanAllByLabelProperty() = default; ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -763,7 +761,7 @@ class ScanAllById : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllById() {} + ScanAllById() = default; ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, storage::View view = storage::View::OLD); @@ -842,7 +840,7 @@ class Expand : public memgraph::query::plan::LogicalOperator { EdgeAtom::Direction direction, const std::vector &edge_types, bool existing_node, storage::View view); - Expand() {} + Expand() = default; bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -950,7 +948,7 @@ class ExpandVariable : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ExpandVariable() {} + ExpandVariable() = default; /** * Creates a variable-length expansion. Most params are forwarded @@ -1073,10 +1071,10 @@ class ConstructNamedPath : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ConstructNamedPath() {} + ConstructNamedPath() = default; ConstructNamedPath(const std::shared_ptr &input, Symbol path_symbol, const std::vector &path_elements) - : input_(input), path_symbol_(path_symbol), path_elements_(path_elements) {} + : input_(input), path_symbol_(std::move(path_symbol)), path_elements_(path_elements) {} bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -1108,13 +1106,13 @@ class Filter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Filter() {} + Filter() = default; Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression); Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression, - const Filters &all_filters); + Filters all_filters); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -1126,12 +1124,12 @@ class Filter : public memgraph::query::plan::LogicalOperator { std::shared_ptr input_; std::vector> pattern_filters_; Expression *expression_; - const memgraph::query::plan::Filters all_filters_; + memgraph::query::plan::Filters all_filters_; static std::string SingleFilterName(const query::plan::FilterInfo &single_filter) { using Type = query::plan::FilterInfo::Type; if (single_filter.type == Type::Generic) { - std::set symbol_names; + std::set> symbol_names; for (const auto &symbol : single_filter.used_symbols) { symbol_names.insert(symbol.name()); } @@ -1144,7 +1142,7 @@ class Filter : public memgraph::query::plan::LogicalOperator { LOG_FATAL("Label filters not using LabelsTest are not supported for query inspection!"); } auto filter_expression = static_cast(single_filter.expression); - std::set label_names; + std::set> label_names; for (const auto &label : filter_expression->labels_) { label_names.insert(label.name); } @@ -1167,7 +1165,7 @@ class Filter : public memgraph::query::plan::LogicalOperator { } std::string ToString() const override { - std::set filter_names; + std::set> filter_names; for (const auto &filter : all_filters_) { filter_names.insert(Filter::SingleFilterName(filter)); } @@ -1214,7 +1212,7 @@ class Produce : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Produce() {} + Produce() = default; Produce(const std::shared_ptr &input, const std::vector &named_expressions); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1271,7 +1269,7 @@ class Delete : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Delete() {} + Delete() = default; Delete(const std::shared_ptr &input_, const std::vector &expressions, bool detach_); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1326,7 +1324,7 @@ class SetProperty : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - SetProperty() {} + SetProperty() = default; SetProperty(const std::shared_ptr &input, storage::PropertyId property, PropertyLookup *lhs, Expression *rhs); @@ -1385,7 +1383,7 @@ class SetProperties : public memgraph::query::plan::LogicalOperator { /// that the old properties are discarded and replaced with new ones. enum class Op { UPDATE, REPLACE }; - SetProperties() {} + SetProperties() = default; SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1433,7 +1431,7 @@ class SetLabels : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - SetLabels() {} + SetLabels() = default; SetLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels); @@ -1477,7 +1475,7 @@ class RemoveProperty : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - RemoveProperty() {} + RemoveProperty() = default; RemoveProperty(const std::shared_ptr &input, storage::PropertyId property, PropertyLookup *lhs); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1522,7 +1520,7 @@ class RemoveLabels : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - RemoveLabels() {} + RemoveLabels() = default; RemoveLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels); @@ -1578,7 +1576,7 @@ class EdgeUniquenessFilter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EdgeUniquenessFilter() {} + EdgeUniquenessFilter() = default; EdgeUniquenessFilter(const std::shared_ptr &input, Symbol expand_symbol, const std::vector &previous_symbols); @@ -1636,7 +1634,7 @@ class EmptyResult : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EmptyResult() {} + EmptyResult() = default; EmptyResult(const std::shared_ptr &input); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1688,7 +1686,7 @@ class Accumulate : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Accumulate() {} + Accumulate() = default; Accumulate(const std::shared_ptr &input, const std::vector &symbols, bool advance_command = false); @@ -1811,7 +1809,7 @@ class Skip : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Skip() {} + Skip() = default; Skip(const std::shared_ptr &input, Expression *expression); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1857,7 +1855,7 @@ class EvaluatePatternFilter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EvaluatePatternFilter() {} + EvaluatePatternFilter() = default; EvaluatePatternFilter(const std::shared_ptr &input, Symbol output_symbol); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1911,7 +1909,7 @@ class Limit : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Limit() {} + Limit() = default; Limit(const std::shared_ptr &input, Expression *expression); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1966,7 +1964,7 @@ class OrderBy : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OrderBy() {} + OrderBy() = default; OrderBy(const std::shared_ptr &input, const std::vector &order_by, const std::vector &output_symbols); @@ -2018,7 +2016,7 @@ class Merge : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Merge() {} + Merge() = default; Merge(const std::shared_ptr &input, const std::shared_ptr &merge_match, const std::shared_ptr &merge_create); @@ -2078,7 +2076,7 @@ class Optional : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Optional() {} + Optional() = default; Optional(const std::shared_ptr &input, const std::shared_ptr &optional, const std::vector &optional_symbols); @@ -2132,7 +2130,7 @@ class Unwind : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Unwind() {} + Unwind() = default; Unwind(const std::shared_ptr &input, Expression *input_expression_, Symbol output_symbol); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2167,7 +2165,7 @@ class Distinct : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Distinct() {} + Distinct() = default; Distinct(const std::shared_ptr &input, const std::vector &value_symbols); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2200,7 +2198,7 @@ class Union : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Union() {} + Union() = default; Union(const std::shared_ptr &left_op, const std::shared_ptr &right_op, const std::vector &union_symbols, const std::vector &left_symbols, @@ -2256,7 +2254,7 @@ class Cartesian : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Cartesian() {} + Cartesian() = default; /** Construct the operator with left input branch and right input branch. */ Cartesian(const std::shared_ptr &left_op, const std::vector &left_symbols, const std::shared_ptr &right_op, const std::vector &right_symbols) @@ -2291,7 +2289,7 @@ class OutputTable : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OutputTable() {} + OutputTable() = default; OutputTable(std::vector output_symbols, std::function>(Frame *, ExecutionContext *)> callback); OutputTable(std::vector output_symbols, std::vector> rows); @@ -2327,7 +2325,7 @@ class OutputTableStream : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OutputTableStream() {} + OutputTableStream() = default; OutputTableStream(std::vector output_symbols, std::function>(Frame *, ExecutionContext *)> callback); @@ -2498,7 +2496,7 @@ class Apply : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Apply() {} + Apply() = default; Apply(const std::shared_ptr input, const std::shared_ptr subquery, bool subquery_has_return); @@ -2545,7 +2543,7 @@ class IndexedJoin : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - IndexedJoin() {} + IndexedJoin() = default; IndexedJoin(std::shared_ptr main_branch, std::shared_ptr sub_branch); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2588,7 +2586,7 @@ class HashJoin : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - HashJoin() {} + HashJoin() = default; /** Construct the operator with left input branch and right input branch. */ HashJoin(const std::shared_ptr &left_op, const std::vector &left_symbols, const std::shared_ptr &right_op, const std::vector &right_symbols, @@ -2631,5 +2629,4 @@ class HashJoin : public memgraph::query::plan::LogicalOperator { }; } // namespace plan -} // namespace query -} // namespace memgraph +} // namespace memgraph::query diff --git a/src/query/plan/planner.hpp b/src/query/plan/planner.hpp index 3f3d853bc..10318e6b9 100644 --- a/src/query/plan/planner.hpp +++ b/src/query/plan/planner.hpp @@ -17,6 +17,8 @@ #pragma once +#include + #include "query/plan/cost_estimator.hpp" #include "query/plan/operator.hpp" #include "query/plan/preprocess.hpp" @@ -42,11 +44,11 @@ class PostProcessor final { using ProcessedPlan = std::unique_ptr; - explicit PostProcessor(const Parameters ¶meters) : parameters_(parameters) {} + explicit PostProcessor(Parameters parameters) : parameters_(std::move(parameters)) {} template - PostProcessor(const Parameters ¶meters, std::vector index_hints, TDbAccessor *db) - : parameters_(parameters), index_hints_(IndexHints(index_hints, db)) {} + PostProcessor(Parameters parameters, std::vector index_hints, TDbAccessor *db) + : parameters_(std::move(parameters)), index_hints_(IndexHints(index_hints, db)) {} template std::unique_ptr Rewrite(std::unique_ptr plan, TPlanningContext *context) { diff --git a/src/query/plan/preprocess.cpp b/src/query/plan/preprocess.cpp index 8b1689796..e03c51841 100644 --- a/src/query/plan/preprocess.cpp +++ b/src/query/plan/preprocess.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "query/exceptions.hpp" @@ -199,7 +200,7 @@ auto SplitExpressionOnAnd(Expression *expression) { PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &symbol, PropertyIx property, Expression *value, Type type) - : symbol_(symbol), property_(property), type_(type), value_(value) { + : symbol_(symbol), property_(std::move(property)), type_(type), value_(value) { MG_ASSERT(type != Type::RANGE); UsedSymbolsCollector collector(symbol_table); value->Accept(collector); @@ -209,7 +210,11 @@ PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &sy PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &symbol, PropertyIx property, const std::optional &lower_bound, const std::optional &upper_bound) - : symbol_(symbol), property_(property), type_(Type::RANGE), lower_bound_(lower_bound), upper_bound_(upper_bound) { + : symbol_(symbol), + property_(std::move(property)), + type_(Type::RANGE), + lower_bound_(lower_bound), + upper_bound_(upper_bound) { UsedSymbolsCollector collector(symbol_table); if (lower_bound) { lower_bound->value()->Accept(collector); @@ -220,8 +225,8 @@ PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &sy is_symbol_in_value_ = utils::Contains(collector.symbols_, symbol); } -PropertyFilter::PropertyFilter(const Symbol &symbol, PropertyIx property, Type type) - : symbol_(symbol), property_(property), type_(type) { +PropertyFilter::PropertyFilter(Symbol symbol, PropertyIx property, Type type) + : symbol_(std::move(symbol)), property_(std::move(property)), type_(type) { // As this constructor is used for property filters where // we don't have to evaluate the filter expression, we set // the is_symbol_in_value_ to false, although the filter diff --git a/src/query/plan/preprocess.hpp b/src/query/plan/preprocess.hpp index e9f7aeeb9..8e1955907 100644 --- a/src/query/plan/preprocess.hpp +++ b/src/query/plan/preprocess.hpp @@ -103,29 +103,29 @@ class UsedSymbolsCollector : public HierarchicalTreeVisitor { }; // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define PREPROCESS_DEFINE_ID_TYPE(name) \ - class name final { \ - private: \ - explicit name(uint64_t id) : id_(id) {} \ - \ - public: \ - /* Default constructor to allow serialization or preallocation. */ \ - name() = default; \ - \ - static name FromUint(uint64_t id) { return name(id); } \ - static name FromInt(int64_t id) { return name(utils::MemcpyCast(id)); } \ - uint64_t AsUint() const { return id_; } \ - int64_t AsInt() const { return utils::MemcpyCast(id_); } \ - \ - private: \ - uint64_t id_; \ - }; \ - static_assert(std::is_trivially_copyable::value, "query::plan::" #name " must be trivially copyable!"); \ - inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ - inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ - inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ - inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \ - inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \ +#define PREPROCESS_DEFINE_ID_TYPE(name) \ + class name final { \ + private: \ + explicit name(uint64_t id) : id_(id) {} \ + \ + public: \ + /* Default constructor to allow serialization or preallocation. */ \ + name() = default; \ + \ + static name FromUint(uint64_t id) { return name(id); } \ + static name FromInt(int64_t id) { return name(utils::MemcpyCast(id)); } \ + uint64_t AsUint() const { return id_; } \ + int64_t AsInt() const { return utils::MemcpyCast(id_); } \ + \ + private: \ + uint64_t id_; \ + }; \ + static_assert(std::is_trivially_copyable_v, "query::plan::" #name " must be trivially copyable!"); \ + inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ + inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ + inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ + inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \ + inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \ inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); } PREPROCESS_DEFINE_ID_TYPE(ExpansionGroupId); @@ -259,7 +259,7 @@ class PropertyFilter { /// Used for the "PROP IS NOT NULL" filter, and can be used for any /// property filter that doesn't need to use an expression to produce /// values that should be filtered further. - PropertyFilter(const Symbol &, PropertyIx, Type); + PropertyFilter(Symbol, PropertyIx, Type); /// Symbol whose property is looked up. Symbol symbol_; diff --git a/src/query/plan/profile.hpp b/src/query/plan/profile.hpp index 041a34ac9..e0d884449 100644 --- a/src/query/plan/profile.hpp +++ b/src/query/plan/profile.hpp @@ -18,9 +18,7 @@ #include "query/typed_value.hpp" -namespace memgraph::query { - -namespace plan { +namespace memgraph::query::plan { /** * Stores profiling statistics for a single logical operator. @@ -43,5 +41,4 @@ std::vector> ProfilingStatsToTable(const ProfilingStatsW nlohmann::json ProfilingStatsToJson(const ProfilingStatsWithTotalTime &stats); -} // namespace plan -} // namespace memgraph::query +} // namespace memgraph::query::plan diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 05361cf73..4054f8c12 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -84,7 +85,7 @@ template class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { public: IndexLookupRewriter(SymbolTable *symbol_table, AstStorage *ast_storage, TDbAccessor *db, IndexHints index_hints) - : symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), index_hints_(index_hints) {} + : symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), index_hints_(std::move(index_hints)) {} using HierarchicalLogicalOperatorVisitor::PostVisit; using HierarchicalLogicalOperatorVisitor::PreVisit; @@ -676,9 +677,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { if (!db_->LabelPropertyIndexExists(GetLabel(label), GetProperty(property))) { continue; } - candidate_indices.emplace_back(std::make_pair( + candidate_indices.emplace_back( IndexHint{.index_type_ = IndexHint::IndexType::LABEL_PROPERTY, .label_ = label, .property_ = property}, - filter)); + filter); candidate_index_lookup.insert({std::make_pair(label, property), filter}); } } diff --git a/src/query/plan/variable_start_planner.cpp b/src/query/plan/variable_start_planner.cpp index eb30226d8..1c230628a 100644 --- a/src/query/plan/variable_start_planner.cpp +++ b/src/query/plan/variable_start_planner.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "utils/flag_validation.hpp" #include "utils/logging.hpp" @@ -216,7 +217,7 @@ CartesianProduct VaryMultiMatchingStarts(const std::vector variants; variants.reserve(matchings.size()); for (const auto &matching : matchings) { - variants.emplace_back(VaryMatchingStart(matching, symbol_table)); + variants.emplace_back(matching, symbol_table); } return MakeCartesianProduct(std::move(variants)); } @@ -247,8 +248,7 @@ VaryQueryPartMatching::VaryQueryPartMatching(SingleQueryPart query_part, const S merge_matchings_(VaryMultiMatchingStarts(query_part_.merge_matching, symbol_table)), filter_matchings_(VaryFilterMatchingStarts(query_part_.matching, symbol_table)) {} -VaryQueryPartMatching::iterator::iterator(const SingleQueryPart &query_part, - VaryMatchingStart::iterator matchings_begin, +VaryQueryPartMatching::iterator::iterator(SingleQueryPart query_part, VaryMatchingStart::iterator matchings_begin, VaryMatchingStart::iterator matchings_end, CartesianProduct::iterator optional_begin, CartesianProduct::iterator optional_end, @@ -256,18 +256,18 @@ VaryQueryPartMatching::iterator::iterator(const SingleQueryPart &query_part, CartesianProduct::iterator merge_end, CartesianProduct::iterator filter_begin, CartesianProduct::iterator filter_end) - : current_query_part_(query_part), - matchings_it_(matchings_begin), - matchings_end_(matchings_end), + : current_query_part_(std::move(query_part)), + matchings_it_(std::move(matchings_begin)), + matchings_end_(std::move(matchings_end)), optional_it_(optional_begin), optional_begin_(optional_begin), - optional_end_(optional_end), + optional_end_(std::move(optional_end)), merge_it_(merge_begin), merge_begin_(merge_begin), - merge_end_(merge_end), + merge_end_(std::move(merge_end)), filter_it_(filter_begin), filter_begin_(filter_begin), - filter_end_(filter_end) { + filter_end_(std::move(filter_end)) { if (matchings_it_ != matchings_end_) { // Fill the query part with the first variation of matchings SetCurrentQueryPart(); diff --git a/src/query/plan/variable_start_planner.hpp b/src/query/plan/variable_start_planner.hpp index 96fd3b78d..6bdba71f3 100644 --- a/src/query/plan/variable_start_planner.hpp +++ b/src/query/plan/variable_start_planner.hpp @@ -49,16 +49,16 @@ class CartesianProduct { using TElement = typename decltype(begin_->begin())::value_type; public: - CartesianProduct(std::vector sets) + explicit CartesianProduct(std::vector sets) : original_sets_(std::move(sets)), begin_(original_sets_.begin()), end_(original_sets_.end()) {} class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef std::vector value_type; - typedef long difference_type; - typedef const std::vector &reference; - typedef const std::vector *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = std::vector; + using difference_type = long; + using reference = const std::vector &; + using pointer = const std::vector *; explicit iterator(CartesianProduct *self, bool is_done) : self_(self), is_done_(is_done) { if (is_done || self->begin_ == self->end_) { @@ -186,11 +186,11 @@ class VaryMatchingStart { class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef Matching value_type; - typedef long difference_type; - typedef const Matching &reference; - typedef const Matching *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = Matching; + using difference_type = long; + using reference = const Matching &; + using pointer = const Matching *; iterator(VaryMatchingStart *, bool); @@ -240,13 +240,13 @@ class VaryQueryPartMatching { class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef SingleQueryPart value_type; - typedef long difference_type; - typedef const SingleQueryPart &reference; - typedef const SingleQueryPart *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = SingleQueryPart; + using difference_type = long; + using reference = const SingleQueryPart &; + using pointer = const SingleQueryPart *; - iterator(const SingleQueryPart &, VaryMatchingStart::iterator, VaryMatchingStart::iterator, + iterator(SingleQueryPart, VaryMatchingStart::iterator, VaryMatchingStart::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator); @@ -383,8 +383,8 @@ class VariableStartPlanner { /// @brief The result of plan generation is an iterable of roots to multiple /// generated operator trees. - using PlanResult = typename std::result_of::Plan)( - VariableStartPlanner, QueryParts &)>::type; + using PlanResult = std::result_of_t::Plan)( + VariableStartPlanner, QueryParts &)>; }; } // namespace memgraph::query::plan diff --git a/src/query/plan/vertex_count_cache.hpp b/src/query/plan/vertex_count_cache.hpp index ff19ee95a..4cfb2486b 100644 --- a/src/query/plan/vertex_count_cache.hpp +++ b/src/query/plan/vertex_count_cache.hpp @@ -27,7 +27,7 @@ namespace memgraph::query::plan { template class VertexCountCache { public: - VertexCountCache(TDbAccessor *db) : db_(db) {} + explicit VertexCountCache(TDbAccessor *db) : db_(db) {} auto NameToLabel(const std::string &name) { return db_->NameToLabel(name); } auto NameToProperty(const std::string &name) { return db_->NameToProperty(name); } @@ -88,7 +88,7 @@ class VertexCountCache { } private: - typedef std::pair LabelPropertyKey; + using LabelPropertyKey = std::pair; struct LabelPropertyHash { size_t operator()(const LabelPropertyKey &key) const { @@ -96,9 +96,8 @@ class VertexCountCache { } }; - typedef std::pair>, - std::optional>> - BoundsKey; + using BoundsKey = std::pair>, + std::optional>>; struct BoundsHash { size_t operator()(const BoundsKey &key) const { diff --git a/src/replication/include/replication/config.hpp b/src/replication/include/replication/config.hpp index ca0cd8f16..f98069955 100644 --- a/src/replication/include/replication/config.hpp +++ b/src/replication/include/replication/config.hpp @@ -34,8 +34,8 @@ struct ReplicationClientConfig { std::chrono::seconds replica_check_frequency{1}; struct SSL { - std::string key_file = ""; - std::string cert_file = ""; + std::string key_file; + std::string cert_file; friend bool operator==(const SSL &, const SSL &) = default; }; diff --git a/src/replication/include/replication/replication_server.hpp b/src/replication/include/replication/replication_server.hpp index 032312dcc..e9ca1b549 100644 --- a/src/replication/include/replication/replication_server.hpp +++ b/src/replication/include/replication/replication_server.hpp @@ -23,7 +23,7 @@ struct FrequentHeartbeatReq { static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); - FrequentHeartbeatReq() {} + FrequentHeartbeatReq() = default; }; struct FrequentHeartbeatRes { @@ -32,7 +32,7 @@ struct FrequentHeartbeatRes { static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); - FrequentHeartbeatRes() {} + FrequentHeartbeatRes() = default; explicit FrequentHeartbeatRes(bool success) : success(success) {} bool success; diff --git a/src/requests/requests.cpp b/src/requests/requests.cpp index d6c37bd61..8a871184a 100644 --- a/src/requests/requests.cpp +++ b/src/requests/requests.cpp @@ -35,7 +35,7 @@ bool RequestPostJson(const std::string &url, const nlohmann::json &data, int tim CURLcode res = CURLE_UNSUPPORTED_PROTOCOL; long response_code = 0; - struct curl_slist *headers = NULL; + struct curl_slist *headers = nullptr; std::string payload = data.dump(); std::string user_agent = fmt::format("memgraph/{}", gflags::VersionString()); diff --git a/src/rpc/client.hpp b/src/rpc/client.hpp index bd98afe89..f727391ac 100644 --- a/src/rpc/client.hpp +++ b/src/rpc/client.hpp @@ -55,7 +55,7 @@ class Client { StreamHandler(const StreamHandler &) = delete; StreamHandler &operator=(const StreamHandler &) = delete; - ~StreamHandler() {} + ~StreamHandler() = default; slk::Builder *GetBuilder() { return &req_builder_; } diff --git a/src/rpc/client_pool.hpp b/src/rpc/client_pool.hpp index 408b006a2..86123899d 100644 --- a/src/rpc/client_pool.hpp +++ b/src/rpc/client_pool.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -13,6 +13,7 @@ #include #include +#include #include "rpc/client.hpp" @@ -25,8 +26,8 @@ namespace memgraph::rpc { */ class ClientPool { public: - ClientPool(const io::network::Endpoint &endpoint, communication::ClientContext *context) - : endpoint_(endpoint), context_(context) {} + ClientPool(io::network::Endpoint endpoint, communication::ClientContext *context) + : endpoint_(std::move(endpoint)), context_(context) {} template typename TRequestResponse::Response Call(Args &&...args) { diff --git a/src/rpc/exceptions.hpp b/src/rpc/exceptions.hpp index b0eb6c329..346c53a9a 100644 --- a/src/rpc/exceptions.hpp +++ b/src/rpc/exceptions.hpp @@ -21,7 +21,7 @@ namespace memgraph::rpc { /// This exception always requires explicit handling. class RpcFailedException : public utils::BasicException { public: - RpcFailedException(std::string_view msg) : utils::BasicException(msg) {} + explicit RpcFailedException(std::string_view msg) : utils::BasicException(msg) {} SPECIALIZE_GET_EXCEPTION_NAME(RpcFailedException); }; diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index 933daaa7f..8bc77579b 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -11,6 +11,8 @@ #include "rpc/protocol.hpp" +#include + #include "rpc/messages.hpp" #include "rpc/server.hpp" #include "rpc/version.hpp" @@ -21,9 +23,9 @@ namespace memgraph::rpc { -Session::Session(Server *server, const io::network::Endpoint &endpoint, communication::InputStream *input_stream, +Session::Session(Server *server, io::network::Endpoint endpoint, communication::InputStream *input_stream, communication::OutputStream *output_stream) - : server_(server), endpoint_(endpoint), input_stream_(input_stream), output_stream_(output_stream) {} + : server_(server), endpoint_(std::move(endpoint)), input_stream_(input_stream), output_stream_(output_stream) {} void Session::Execute() { auto ret = slk::CheckStreamComplete(input_stream_->data(), input_stream_->size()); diff --git a/src/rpc/protocol.hpp b/src/rpc/protocol.hpp index f8b25664c..a1deea17c 100644 --- a/src/rpc/protocol.hpp +++ b/src/rpc/protocol.hpp @@ -48,7 +48,7 @@ class SessionException : public utils::BasicException { */ class Session { public: - Session(Server *server, const io::network::Endpoint &endpoint, communication::InputStream *input_stream, + Session(Server *server, io::network::Endpoint endpoint, communication::InputStream *input_stream, communication::OutputStream *output_stream); /** diff --git a/src/slk/serialization.hpp b/src/slk/serialization.hpp index 06628e229..9ca99527d 100644 --- a/src/slk/serialization.hpp +++ b/src/slk/serialization.hpp @@ -60,10 +60,10 @@ void Save(const std::vector &obj, Builder *builder); template void Load(std::vector *obj, Reader *reader); -template -void Save(const std::set &obj, Builder *builder); -template -void Load(std::set *obj, Reader *reader); +template +void Save(const std::set &obj, Builder *builder); +template +void Load(std::set *obj, Reader *reader); template void Save(const std::map &obj, Builder *builder); @@ -201,8 +201,8 @@ inline void Load(std::vector *obj, Reader *reader) { } } -template -inline void Save(const std::set &obj, Builder *builder) { +template +inline void Save(const std::set &obj, Builder *builder) { uint64_t size = obj.size(); Save(size, builder); for (const auto &item : obj) { @@ -210,8 +210,8 @@ inline void Save(const std::set &obj, Builder *builder) { } } -template -inline void Load(std::set *obj, Reader *reader) { +template +inline void Load(std::set *obj, Reader *reader) { uint64_t size = 0; Load(&size, reader); for (uint64_t i = 0; i < size; ++i) { @@ -273,7 +273,7 @@ inline void Load(std::unique_ptr *obj, Reader *reader) { // Prevent any loading which may potentially break class hierarchies. // Unfortunately, C++14 doesn't have (or I'm not aware of it) a trait for // checking whether some type has any derived or base classes. - static_assert(!std::is_polymorphic::value, + static_assert(!std::is_polymorphic_v, "Only non polymorphic types can be loaded generically from a " "pointer. Pass a custom load function as the 3rd argument."); bool exists = false; @@ -379,7 +379,7 @@ inline void Load(std::shared_ptr *obj, Reader *reader, std::vector::value, + static_assert(!std::is_polymorphic_v, "Only non polymorphic types can be loaded generically from a " "pointer. Pass a custom load function as the 4th argument."); bool exists = false; diff --git a/src/slk/streams.cpp b/src/slk/streams.cpp index 1346393e8..5125d635a 100644 --- a/src/slk/streams.cpp +++ b/src/slk/streams.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -12,12 +12,13 @@ #include "slk/streams.hpp" #include +#include #include "utils/logging.hpp" namespace memgraph::slk { -Builder::Builder(std::function write_func) : write_func_(write_func) {} +Builder::Builder(std::function write_func) : write_func_(std::move(write_func)) {} void Builder::Save(const uint8_t *data, uint64_t size) { size_t offset = 0; diff --git a/src/storage/v2/disk/label_property_index.cpp b/src/storage/v2/disk/label_property_index.cpp index 5e538559a..9a40f03d1 100644 --- a/src/storage/v2/disk/label_property_index.cpp +++ b/src/storage/v2/disk/label_property_index.cpp @@ -211,8 +211,7 @@ uint64_t DiskLabelPropertyIndex::ApproximateVertexCount( void DiskLabelPropertyIndex::LoadIndexInfo(const std::vector &keys) { for (const auto &label_property : keys) { std::vector label_property_split = utils::Split(label_property, ","); - index_.emplace( - std::make_pair(LabelId::FromString(label_property_split[0]), PropertyId::FromString(label_property_split[1]))); + index_.emplace(LabelId::FromString(label_property_split[0]), PropertyId::FromString(label_property_split[1])); } } diff --git a/src/storage/v2/disk/rocksdb_storage.hpp b/src/storage/v2/disk/rocksdb_storage.hpp index 0e55c2748..09200d38a 100644 --- a/src/storage/v2/disk/rocksdb_storage.hpp +++ b/src/storage/v2/disk/rocksdb_storage.hpp @@ -32,7 +32,7 @@ namespace memgraph::storage { /// Wraps RocksDB objects inside a struct. Vertex_chandle and edge_chandle are column family handles that may be /// nullptr. In that case client should take care about them. struct RocksDBStorage { - explicit RocksDBStorage() {} + explicit RocksDBStorage() = default; RocksDBStorage(const RocksDBStorage &) = delete; RocksDBStorage &operator=(const RocksDBStorage &) = delete; diff --git a/src/storage/v2/disk/unique_constraints.cpp b/src/storage/v2/disk/unique_constraints.cpp index e0ec3cf82..3c17530c2 100644 --- a/src/storage/v2/disk/unique_constraints.cpp +++ b/src/storage/v2/disk/unique_constraints.cpp @@ -344,7 +344,7 @@ void DiskUniqueConstraints::LoadUniqueConstraints(const std::vector for (int i = 1; i < key_parts.size(); i++) { properties.insert(PropertyId::FromString(key_parts[i])); } - constraints_.emplace(std::make_pair(label, properties)); + constraints_.emplace(label, properties); } } diff --git a/src/storage/v2/durability/serialization.hpp b/src/storage/v2/durability/serialization.hpp index 409293220..ca3a19e0f 100644 --- a/src/storage/v2/durability/serialization.hpp +++ b/src/storage/v2/durability/serialization.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -27,7 +27,7 @@ namespace memgraph::storage::durability { /// (e.g. file and network). class BaseEncoder { protected: - ~BaseEncoder() {} + ~BaseEncoder() = default; public: virtual void WriteMarker(Marker marker) = 0; @@ -84,7 +84,7 @@ class Encoder final : public BaseEncoder { /// (e.g. file and network). class BaseDecoder { protected: - ~BaseDecoder() {} + ~BaseDecoder() = default; public: virtual std::optional ReadMarker() = 0; diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index d4278dba9..52872222b 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -1212,7 +1212,7 @@ RecoveredSnapshot LoadSnapshotVersion15(const std::filesystem::path &path, utils spdlog::info("Recover connectivity."); recovery_info.vertex_batches.reserve(vertex_batches.size()); for (const auto batch : vertex_batches) { - recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count)); + recovery_info.vertex_batches.emplace_back(Gid::FromUint(0), batch.count); } std::atomic highest_edge_gid{0}; @@ -1505,7 +1505,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis spdlog::info("Recover connectivity."); recovery_info.vertex_batches.reserve(vertex_batches.size()); for (const auto batch : vertex_batches) { - recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count)); + recovery_info.vertex_batches.emplace_back(Gid::FromUint(0), batch.count); } std::atomic highest_edge_gid{0}; diff --git a/src/storage/v2/durability/wal.hpp b/src/storage/v2/durability/wal.hpp index 3f598cb99..8f6492ac7 100644 --- a/src/storage/v2/durability/wal.hpp +++ b/src/storage/v2/durability/wal.hpp @@ -108,7 +108,7 @@ struct WalDeltaData { struct { std::string label; - std::set properties; + std::set> properties; } operation_label_properties; struct { diff --git a/src/storage/v2/edge_accessor.hpp b/src/storage/v2/edge_accessor.hpp index 86912c2b6..365619149 100644 --- a/src/storage/v2/edge_accessor.hpp +++ b/src/storage/v2/edge_accessor.hpp @@ -110,7 +110,7 @@ class EdgeAccessor final { } // namespace memgraph::storage -static_assert(std::is_trivially_copyable::value, +static_assert(std::is_trivially_copyable_v, "storage::EdgeAccessor must be trivially copyable!"); namespace std { diff --git a/src/storage/v2/id_types.hpp b/src/storage/v2/id_types.hpp index 2f9577246..3f2c8aa40 100644 --- a/src/storage/v2/id_types.hpp +++ b/src/storage/v2/id_types.hpp @@ -42,7 +42,7 @@ namespace memgraph::storage { private: \ uint64_t id_; \ }; \ - static_assert(std::is_trivially_copyable::value, "storage::" #name " must be trivially copyable!"); \ + static_assert(std::is_trivially_copyable_v, "storage::" #name " must be trivially copyable!"); \ inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ diff --git a/src/storage/v2/modified_edge.hpp b/src/storage/v2/modified_edge.hpp index e08ef4ce8..293c74f43 100644 --- a/src/storage/v2/modified_edge.hpp +++ b/src/storage/v2/modified_edge.hpp @@ -34,8 +34,7 @@ struct ModifiedEdgeInfo { EdgeRef edge_ref; }; -static_assert(std::is_trivially_copyable::value, - "storage::ModifiedEdgeInfo must be trivially copyable!"); +static_assert(std::is_trivially_copyable_v, "storage::ModifiedEdgeInfo must be trivially copyable!"); using ModifiedEdgesMap = std::unordered_map; diff --git a/src/storage/v2/property_store.cpp b/src/storage/v2/property_store.cpp index 65f08f4d9..530d5f5f6 100644 --- a/src/storage/v2/property_store.cpp +++ b/src/storage/v2/property_store.cpp @@ -179,7 +179,7 @@ class Writer { public: class MetadataHandle { public: - MetadataHandle() {} + MetadataHandle() = default; explicit MetadataHandle(uint8_t *value) : value_(value) {} @@ -195,7 +195,7 @@ class Writer { uint8_t *value_{nullptr}; }; - Writer() {} + Writer() = default; Writer(uint8_t *data, uint64_t size) : data_(data), size_(size) {} @@ -1311,7 +1311,7 @@ std::vector> PropertyStore: id_old_new_change.reserve(properties.size() + old_properties.size()); for (const auto &[prop_id, new_value] : properties) { if (!old_properties.contains(prop_id)) { - id_old_new_change.emplace_back(std::make_tuple(prop_id, PropertyValue(), new_value)); + id_old_new_change.emplace_back(prop_id, PropertyValue(), new_value); } } @@ -1319,7 +1319,7 @@ std::vector> PropertyStore: auto [it, inserted] = properties.emplace(old_key, old_value); if (!inserted) { auto &new_value = it->second; - id_old_new_change.emplace_back(std::make_tuple(it->first, old_value, new_value)); + id_old_new_change.emplace_back(it->first, old_value, new_value); } } diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index 817f47bcb..8cd8cb384 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -90,7 +90,7 @@ class ReplicationClient { auto GetStorageId() const -> std::string; void Start(); - void StartTransactionReplication(const uint64_t current_wal_seq_num); + void StartTransactionReplication(uint64_t current_wal_seq_num); // Replication clients can be removed at any point // so to avoid any complexity of checking if the client was removed whenever // we want to send part of transaction and to avoid adding some GC logic this diff --git a/src/storage/v2/replication/rpc.cpp b/src/storage/v2/replication/rpc.cpp index 1477aa108..b722dfebf 100644 --- a/src/storage/v2/replication/rpc.cpp +++ b/src/storage/v2/replication/rpc.cpp @@ -14,9 +14,7 @@ namespace memgraph { -namespace storage { - -namespace replication { +namespace storage::replication { void AppendDeltasReq::Save(const AppendDeltasReq &self, memgraph::slk::Builder *builder) { memgraph::slk::Save(self, builder); @@ -59,8 +57,7 @@ void TimestampRes::Save(const TimestampRes &self, memgraph::slk::Builder *builde } void TimestampRes::Load(TimestampRes *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); } -} // namespace replication -} // namespace storage +} // namespace storage::replication constexpr utils::TypeInfo storage::replication::AppendDeltasReq::kType{utils::TypeId::REP_APPEND_DELTAS_REQ, "AppendDeltasReq", nullptr}; diff --git a/src/storage/v2/replication/rpc.hpp b/src/storage/v2/replication/rpc.hpp index 1c0d425c8..9e2f0b35e 100644 --- a/src/storage/v2/replication/rpc.hpp +++ b/src/storage/v2/replication/rpc.hpp @@ -14,16 +14,13 @@ #include #include #include +#include #include "rpc/messages.hpp" #include "slk/serialization.hpp" #include "slk/streams.hpp" -namespace memgraph { - -namespace storage { - -namespace replication { +namespace memgraph::storage::replication { struct AppendDeltasReq { static const utils::TypeInfo kType; @@ -31,7 +28,7 @@ struct AppendDeltasReq { static void Load(AppendDeltasReq *self, memgraph::slk::Reader *reader); static void Save(const AppendDeltasReq &self, memgraph::slk::Builder *builder); - AppendDeltasReq() {} + AppendDeltasReq() = default; AppendDeltasReq(std::string name, uint64_t previous_commit_timestamp, uint64_t seq_num) : db_name(std::move(name)), previous_commit_timestamp(previous_commit_timestamp), seq_num(seq_num) {} @@ -46,7 +43,7 @@ struct AppendDeltasRes { static void Load(AppendDeltasRes *self, memgraph::slk::Reader *reader); static void Save(const AppendDeltasRes &self, memgraph::slk::Builder *builder); - AppendDeltasRes() {} + AppendDeltasRes() = default; AppendDeltasRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -63,7 +60,7 @@ struct HeartbeatReq { static void Load(HeartbeatReq *self, memgraph::slk::Reader *reader); static void Save(const HeartbeatReq &self, memgraph::slk::Builder *builder); - HeartbeatReq() {} + HeartbeatReq() = default; HeartbeatReq(std::string name, uint64_t main_commit_timestamp, std::string epoch_id) : db_name(std::move(name)), main_commit_timestamp(main_commit_timestamp), epoch_id(std::move(epoch_id)) {} @@ -78,12 +75,12 @@ struct HeartbeatRes { static void Load(HeartbeatRes *self, memgraph::slk::Reader *reader); static void Save(const HeartbeatRes &self, memgraph::slk::Builder *builder); - HeartbeatRes() {} + HeartbeatRes() = default; HeartbeatRes(std::string name, bool success, uint64_t current_commit_timestamp, std::string epoch_id) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp), - epoch_id(epoch_id) {} + epoch_id(std::move(epoch_id)) {} std::string db_name; bool success; @@ -99,7 +96,7 @@ struct SnapshotReq { static void Load(SnapshotReq *self, memgraph::slk::Reader *reader); static void Save(const SnapshotReq &self, memgraph::slk::Builder *builder); - SnapshotReq() {} + SnapshotReq() = default; explicit SnapshotReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -111,7 +108,7 @@ struct SnapshotRes { static void Load(SnapshotRes *self, memgraph::slk::Reader *reader); static void Save(const SnapshotRes &self, memgraph::slk::Builder *builder); - SnapshotRes() {} + SnapshotRes() = default; SnapshotRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -128,7 +125,7 @@ struct WalFilesReq { static void Load(WalFilesReq *self, memgraph::slk::Reader *reader); static void Save(const WalFilesReq &self, memgraph::slk::Builder *builder); - WalFilesReq() {} + WalFilesReq() = default; explicit WalFilesReq(std::string name, uint64_t file_number) : db_name(std::move(name)), file_number(file_number) {} std::string db_name; @@ -141,7 +138,7 @@ struct WalFilesRes { static void Load(WalFilesRes *self, memgraph::slk::Reader *reader); static void Save(const WalFilesRes &self, memgraph::slk::Builder *builder); - WalFilesRes() {} + WalFilesRes() = default; WalFilesRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -158,7 +155,7 @@ struct CurrentWalReq { static void Load(CurrentWalReq *self, memgraph::slk::Reader *reader); static void Save(const CurrentWalReq &self, memgraph::slk::Builder *builder); - CurrentWalReq() {} + CurrentWalReq() = default; explicit CurrentWalReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -170,7 +167,7 @@ struct CurrentWalRes { static void Load(CurrentWalRes *self, memgraph::slk::Reader *reader); static void Save(const CurrentWalRes &self, memgraph::slk::Builder *builder); - CurrentWalRes() {} + CurrentWalRes() = default; CurrentWalRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -187,7 +184,7 @@ struct TimestampReq { static void Load(TimestampReq *self, memgraph::slk::Reader *reader); static void Save(const TimestampReq &self, memgraph::slk::Builder *builder); - TimestampReq() {} + TimestampReq() = default; explicit TimestampReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -199,7 +196,7 @@ struct TimestampRes { static void Load(TimestampRes *self, memgraph::slk::Reader *reader); static void Save(const TimestampRes &self, memgraph::slk::Builder *builder); - TimestampRes() {} + TimestampRes() = default; TimestampRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -209,12 +206,9 @@ struct TimestampRes { }; using TimestampRpc = rpc::RequestResponse; -} // namespace replication -} // namespace storage -} // namespace memgraph +} // namespace memgraph::storage::replication // SLK serialization declarations -#include "slk/serialization.hpp" namespace memgraph::slk { void Save(const memgraph::storage::replication::TimestampRes &self, memgraph::slk::Builder *builder); diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index d91846ca4..8d8b06cd6 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -140,7 +140,7 @@ class Storage { Accessor(Accessor &&other) noexcept; - virtual ~Accessor() {} + virtual ~Accessor() = default; virtual VertexAccessor CreateVertex() = 0; diff --git a/src/storage/v2/vertex_accessor.hpp b/src/storage/v2/vertex_accessor.hpp index 8f67bc30b..0e5972d14 100644 --- a/src/storage/v2/vertex_accessor.hpp +++ b/src/storage/v2/vertex_accessor.hpp @@ -127,7 +127,7 @@ class VertexAccessor final { bool for_deleted_{false}; }; -static_assert(std::is_trivially_copyable::value, +static_assert(std::is_trivially_copyable_v, "storage::VertexAccessor must be trivially copyable!"); struct EdgesVertexAccessorResult { diff --git a/src/telemetry/telemetry.cpp b/src/telemetry/telemetry.cpp index 714788841..1635dda99 100644 --- a/src/telemetry/telemetry.cpp +++ b/src/telemetry/telemetry.cpp @@ -12,6 +12,7 @@ #include "telemetry/telemetry.hpp" #include +#include #include @@ -36,8 +37,8 @@ Telemetry::Telemetry(std::string url, std::filesystem::path storage_directory, s bool ssl, std::filesystem::path root_directory, std::chrono::duration refresh_interval, const uint64_t send_every_n) : url_(std::move(url)), - uuid_(uuid), - machine_id_(machine_id), + uuid_(std::move(uuid)), + machine_id_(std::move(machine_id)), ssl_(ssl), send_every_n_(send_every_n), storage_(std::move(storage_directory)) { diff --git a/src/utils/cast.hpp b/src/utils/cast.hpp index bdb8facc6..f015caf94 100644 --- a/src/utils/cast.hpp +++ b/src/utils/cast.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -18,8 +18,8 @@ namespace memgraph::utils { template -constexpr typename std::underlying_type::type UnderlyingCast(T e) { - return static_cast::type>(e); +constexpr std::underlying_type_t UnderlyingCast(T e) { + return static_cast>(e); } /** @@ -36,8 +36,8 @@ template TDest MemcpyCast(TSrc src) { TDest dest; static_assert(sizeof(dest) == sizeof(src), "MemcpyCast expects source and destination to be of same size"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects source is an arithmetic type"); - static_assert(std::is_arithmetic::value, "MemcypCast expects destination is an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects source is an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcypCast expects destination is an arithmetic type"); std::memcpy(&dest, &src, sizeof(src)); return dest; } diff --git a/src/utils/event_histogram.hpp b/src/utils/event_histogram.hpp index 3768f0ec1..02d612b60 100644 --- a/src/utils/event_histogram.hpp +++ b/src/utils/event_histogram.hpp @@ -12,6 +12,7 @@ #pragma once #include +#include #include "utils/logging.hpp" @@ -73,7 +74,9 @@ class Histogram { percentiles_ = {0, 25, 50, 75, 90, 100}; } - explicit Histogram(std::vector percentiles) : percentiles_(percentiles) { samples_.resize(kSampleLimit, 0); } + explicit Histogram(std::vector percentiles) : percentiles_(std::move(percentiles)) { + samples_.resize(kSampleLimit, 0); + } uint64_t Count() const { return count_.load(std::memory_order_relaxed); } @@ -104,7 +107,7 @@ class Histogram { percentile_yield.reserve(percentiles_.size()); for (const auto percentile : percentiles_) { - percentile_yield.emplace_back(std::make_pair(percentile, Percentile(percentile))); + percentile_yield.emplace_back(percentile, Percentile(percentile)); } return percentile_yield; diff --git a/src/utils/exceptions.hpp b/src/utils/exceptions.hpp index d9a927beb..fa49f770e 100644 --- a/src/utils/exceptions.hpp +++ b/src/utils/exceptions.hpp @@ -61,7 +61,7 @@ class BasicException : public std::exception { /** * @brief Virtual destructor to allow for subclassing. */ - virtual ~BasicException() {} + ~BasicException() override = default; /** * @brief Returns a pointer to the (constant) error description. @@ -116,7 +116,7 @@ class StacktraceException : public std::exception { /** * @brief Virtual destructor to allow for subclassing. */ - virtual ~StacktraceException() {} + ~StacktraceException() override = default; /** * @brief Returns a pointer to the (constant) error description. diff --git a/src/utils/logging.hpp b/src/utils/logging.hpp index 9b9e7705b..02389beab 100644 --- a/src/utils/logging.hpp +++ b/src/utils/logging.hpp @@ -47,17 +47,21 @@ inline void AssertFailed(const char *file_name, int line_num, const char *expr, #define GET_MESSAGE(...) \ BOOST_PP_IF(BOOST_PP_EQUAL(BOOST_PP_VARIADIC_SIZE(__VA_ARGS__), 0), "", fmt::format(__VA_ARGS__)) -#define MG_ASSERT(expr, ...) \ - if (expr) [[likely]] { \ - (void)0; \ - } else { \ - ::memgraph::logging::AssertFailed(__FILE__, __LINE__, #expr, GET_MESSAGE(__VA_ARGS__)); \ - } +#define MG_ASSERT(expr, ...) \ + do { \ + if (expr) [[likely]] { \ + (void)0; \ + } else { \ + ::memgraph::logging::AssertFailed(__FILE__, __LINE__, #expr, GET_MESSAGE(__VA_ARGS__)); \ + } \ + } while (false) #ifndef NDEBUG #define DMG_ASSERT(expr, ...) MG_ASSERT(expr, __VA_ARGS__) #else -#define DMG_ASSERT(...) +#define DMG_ASSERT(...) \ + do { \ + } while (false) #endif template @@ -75,7 +79,9 @@ void Fatal(const char *msg, const Args &...msg_args) { #ifndef NDEBUG #define DLOG_FATAL(...) LOG_FATAL(__VA_ARGS__) #else -#define DLOG_FATAL(...) +#define DLOG_FATAL(...) \ + do { \ + } while (false) #endif inline void RedirectToStderr() { spdlog::set_default_logger(spdlog::stderr_color_mt("stderr")); } diff --git a/src/utils/lru_cache.hpp b/src/utils/lru_cache.hpp index 1ab636670..f94365331 100644 --- a/src/utils/lru_cache.hpp +++ b/src/utils/lru_cache.hpp @@ -24,7 +24,7 @@ namespace memgraph::utils { template class LRUCache { public: - LRUCache(int cache_size_) : cache_size(cache_size_){}; + explicit LRUCache(int cache_size_) : cache_size(cache_size_){}; void put(const TKey &key, const TVal &val) { auto it = item_map.find(key); diff --git a/src/utils/memory.hpp b/src/utils/memory.hpp index 3bec47ec0..225a3b6a1 100644 --- a/src/utils/memory.hpp +++ b/src/utils/memory.hpp @@ -45,7 +45,7 @@ class BadAlloc final : public std::bad_alloc { std::string msg_; public: - explicit BadAlloc(const std::string &msg) : msg_(msg) {} + explicit BadAlloc(std::string msg) : msg_(std::move(msg)) {} const char *what() const noexcept override { return msg_.c_str(); } }; @@ -53,7 +53,7 @@ class BadAlloc final : public std::bad_alloc { /// Abstract class for writing custom memory management, i.e. allocators. class MemoryResource { public: - virtual ~MemoryResource() {} + virtual ~MemoryResource() = default; /// Allocate storage with a size of at least `bytes` bytes. /// diff --git a/src/utils/scheduler.hpp b/src/utils/scheduler.hpp index b97320c21..d96178598 100644 --- a/src/utils/scheduler.hpp +++ b/src/utils/scheduler.hpp @@ -28,7 +28,7 @@ namespace memgraph::utils { */ class Scheduler { public: - Scheduler() {} + Scheduler() = default; /** * @param pause - Duration between two function executions. If function is * still running when it should be ran again, it will run right after it diff --git a/src/utils/skip_list.hpp b/src/utils/skip_list.hpp index e2079b5be..de4892375 100644 --- a/src/utils/skip_list.hpp +++ b/src/utils/skip_list.hpp @@ -1091,8 +1091,8 @@ class SkipList final : detail::SkipListNode_base { if (lower) { layer_found = find_node(lower->value(), preds, succs); } else { - for (int i = 0; i < kSkipListMaxHeight; ++i) { - preds[i] = head_; + for (auto &pred : preds) { + pred = head_; } layer_found = kSkipListMaxHeight - 1; } diff --git a/src/utils/small_vector.hpp b/src/utils/small_vector.hpp index 8654c1849..d6d7a8dd2 100644 --- a/src/utils/small_vector.hpp +++ b/src/utils/small_vector.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -105,12 +105,12 @@ class SmallVectorTemplateCommon : public SmallVectorBase { // don't want it to be automatically run, so we need to represent the space as // something else. Use an array of char of sufficient alignment. ////////////typedef utils::AlignedCharArrayUnion U; - typedef typename std::aligned_union<1, T>::type U; + using U = typename std::aligned_union<1, T>::type; U first_el_; // Space after 'first_el' is clobbered, do not add any instance vars after it. protected: - SmallVectorTemplateCommon(size_t size) : SmallVectorBase(&first_el_, size) {} + explicit SmallVectorTemplateCommon(size_t size) : SmallVectorBase(&first_el_, size) {} void GrowPod(size_t min_size_in_bytes, size_t t_size) { SmallVectorBase::GrowPod(&first_el_, min_size_in_bytes, t_size); @@ -126,19 +126,19 @@ class SmallVectorTemplateCommon : public SmallVectorBase { void SetEnd(T *P) { this->end_x_ = P; } public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T *iterator; - typedef const T *const_iterator; + using size_type = size_t; + using difference_type = ptrdiff_t; + using value_type = T; + using iterator = T *; + using const_iterator = const T *; - typedef std::reverse_iterator const_reverse_iterator; - typedef std::reverse_iterator reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using reverse_iterator = std::reverse_iterator; - typedef T &reference; - typedef const T &const_reference; - typedef T *pointer; - typedef const T *const_pointer; + using reference = T &; + using const_reference = const T &; + using pointer = T *; + using const_pointer = const T *; // forward iterator creation methods. inline iterator begin() { return (iterator)this->begin_x_; } @@ -201,7 +201,7 @@ class SmallVectorTemplateCommon : public SmallVectorBase { template class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: - SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} + explicit SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} static void DestroyRange(T *s, T *e) { while (s != e) { @@ -277,7 +277,7 @@ void SmallVectorTemplateBase::Grow(size_t min_size) { template class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: - SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} + explicit SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} // No need to do a destroy loop for POD's. static void DestroyRange(T *, T *) {} @@ -331,14 +331,14 @@ inline constexpr bool is_pod = std::is_standard_layout_v &&std::is_trivial_v< /// reduce code duplication based on the SmallVector 'n' template parameter. template class SmallVectorImpl : public SmallVectorTemplateBase> { - typedef SmallVectorTemplateBase> SuperClass; + using SuperClass = SmallVectorTemplateBase>; SmallVectorImpl(const SmallVectorImpl &) = delete; public: - typedef typename SuperClass::iterator iterator; - typedef typename SuperClass::const_iterator const_iterator; - typedef typename SuperClass::size_type size_type; + using iterator = typename SuperClass::iterator; + using const_iterator = typename SuperClass::const_iterator; + using size_type = typename SuperClass::size_type; protected: // Default ctor - Initialize to empty. @@ -861,7 +861,7 @@ class SmallVector : public SmallVectorImpl { return *this; } - SmallVector(SmallVectorImpl &&rhs) : SmallVectorImpl(N) { + explicit SmallVector(SmallVectorImpl &&rhs) : SmallVectorImpl(N) { if (!rhs.empty()) SmallVectorImpl::operator=(::std::move(rhs)); } diff --git a/src/utils/stacktrace.hpp b/src/utils/stacktrace.hpp index a2aa5a6c3..f06152e7d 100644 --- a/src/utils/stacktrace.hpp +++ b/src/utils/stacktrace.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -15,6 +15,7 @@ #include #include #include +#include #include "utils/on_scope_exit.hpp" @@ -25,10 +26,10 @@ class Stacktrace { class Line { public: // cppcheck-suppress noExplicitConstructor - Line(const std::string &original) : original(original) {} + explicit Line(std::string original) : original(std::move(original)) {} - Line(const std::string &original, const std::string &function, const std::string &location) - : original(original), function(function), location(location) {} + Line(std::string original, std::string function, std::string location) + : original(std::move(original)), function(std::move(function)), location(std::move(location)) {} std::string original, function, location; }; @@ -84,7 +85,7 @@ class Stacktrace { auto begin = line.find('('); auto end = line.find('+'); - if (begin == std::string::npos || end == std::string::npos) return {original}; + if (begin == std::string::npos || end == std::string::npos) return Line{original}; line[end] = '\0'; diff --git a/src/utils/variant_helpers.hpp b/src/utils/variant_helpers.hpp index 6a58deafb..b7751fe3e 100644 --- a/src/utils/variant_helpers.hpp +++ b/src/utils/variant_helpers.hpp @@ -26,7 +26,7 @@ Overloaded(Ts...) -> Overloaded; template struct ChainedOverloaded : Ts... { template - ChainedOverloaded(Us &&...ts) : Ts(std::forward(ts))... {} + explicit ChainedOverloaded(Us &&...ts) : Ts(std::forward(ts))... {} template auto operator()(Args &&...args) { diff --git a/tests/benchmark/query/profile.cpp b/tests/benchmark/query/profile.cpp index cfa346fbf..1c28fd21c 100644 --- a/tests/benchmark/query/profile.cpp +++ b/tests/benchmark/query/profile.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -233,9 +233,9 @@ struct ScopedProfile { stats = nullptr; // Was this logical operator already hit on one of the previous pulls? - for (size_t i = 0; i < root->children.size(); ++i) { - if (root->children[i].key == key) { - stats = &root->children[i]; + for (auto &child : root->children) { + if (child.key == key) { + stats = &child; break; } } @@ -345,9 +345,9 @@ struct ScopedProfile { stats = nullptr; // Was this logical operator already hit on one of the previous pulls? - for (size_t i = 0; i < root->children.size(); ++i) { - if (root->children[i].key == key) { - stats = &root->children[i]; + for (auto &child : root->children) { + if (child.key == key) { + stats = &child; break; } } diff --git a/tests/benchmark/rpc.cpp b/tests/benchmark/rpc.cpp index 3dace0531..0bb1da4d3 100644 --- a/tests/benchmark/rpc.cpp +++ b/tests/benchmark/rpc.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -24,8 +25,8 @@ struct EchoMessage { static const memgraph::utils::TypeInfo kType; - EchoMessage() {} // Needed for serialization. - explicit EchoMessage(const std::string &data) : data(data) {} + EchoMessage() = default; // Needed for serialization. + explicit EchoMessage(std::string data) : data(std::move(data)) {} static void Load(EchoMessage *obj, memgraph::slk::Reader *reader); static void Save(const EchoMessage &obj, memgraph::slk::Builder *builder); diff --git a/tests/e2e/isolation_levels/isolation_levels.cpp b/tests/e2e/isolation_levels/isolation_levels.cpp index 15151b106..2ead05750 100644 --- a/tests/e2e/isolation_levels/isolation_levels.cpp +++ b/tests/e2e/isolation_levels/isolation_levels.cpp @@ -91,7 +91,7 @@ void SwitchToSameDB(std::unique_ptr &main, std::unique_ptrFetchAll(); MG_ASSERT(dbs, "Failed to show databases"); for (const auto &elem : *dbs) { - MG_ASSERT(elem.size(), "Show databases wrong output"); + MG_ASSERT(!elem.empty(), "Show databases wrong output"); const auto &active = elem[1].ValueString(); if (active == "*") { const auto &name = elem[0].ValueString(); diff --git a/tests/e2e/memory/memory_limit_global_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_alloc_proc.cpp index e1f530123..34aeae509 100644 --- a/tests/e2e/memory/memory_limit_global_alloc_proc.cpp +++ b/tests/e2e/memory/memory_limit_global_alloc_proc.cpp @@ -57,6 +57,6 @@ int main(int argc, char **argv) { MG_ASSERT(client->Execute("CALL libglobal_memory_limit_proc.success() YIELD *")); auto result2 = client->FetchAll(); - MG_ASSERT(result2 != std::nullopt && result2->size() > 0); + MG_ASSERT(result2 != std::nullopt && !result2->empty()); return 0; } diff --git a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp index 2132fbb16..e44c91ea7 100644 --- a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp +++ b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp @@ -54,7 +54,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp index 0c2eb1ee6..e9892ac82 100644 --- a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp +++ b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp @@ -55,7 +55,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/procedure_memory_limit.cpp b/tests/e2e/memory/procedure_memory_limit.cpp index 89a91a9ce..07fbd17b2 100644 --- a/tests/e2e/memory/procedure_memory_limit.cpp +++ b/tests/e2e/memory/procedure_memory_limit.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/query_memory_limit_proc.cpp b/tests/e2e/memory/query_memory_limit_proc.cpp index 39baf4d8e..38509a766 100644 --- a/tests/e2e/memory/query_memory_limit_proc.cpp +++ b/tests/e2e/memory/query_memory_limit_proc.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp b/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp index 5a5ec94f0..5acac5404 100644 --- a/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp +++ b/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/replication/common.hpp b/tests/e2e/replication/common.hpp index f3cff293e..f5113ac37 100644 --- a/tests/e2e/replication/common.hpp +++ b/tests/e2e/replication/common.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -39,7 +39,7 @@ auto ParseDatabaseEndpoints(const std::string &database_endpoints_str) { for (const auto &db_endpoint_str : db_endpoints_strs) { const auto maybe_host_port = memgraph::io::network::Endpoint::ParseSocketOrIpAddress(db_endpoint_str, 7687); MG_ASSERT(maybe_host_port); - database_endpoints.emplace_back(memgraph::io::network::Endpoint(maybe_host_port->first, maybe_host_port->second)); + database_endpoints.emplace_back(maybe_host_port->first, maybe_host_port->second); } return database_endpoints; } diff --git a/tests/e2e/replication/constraints.cpp b/tests/e2e/replication/constraints.cpp index f56e97946..01c1217f2 100644 --- a/tests/e2e/replication/constraints.cpp +++ b/tests/e2e/replication/constraints.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -132,7 +132,7 @@ int main(int argc, char **argv) { auto client = mg::e2e::replication::Connect(database_endpoint); client->Execute("SHOW CONSTRAINT INFO;"); if (const auto data = client->FetchAll()) { - if ((*data).size() != 0) { + if (!(*data).empty()) { LOG_FATAL("{} still have some constraints.", database_endpoint); } } else { diff --git a/tests/e2e/replication/read_write_benchmark.cpp b/tests/e2e/replication/read_write_benchmark.cpp index df1c4fb90..243aab2a8 100644 --- a/tests/e2e/replication/read_write_benchmark.cpp +++ b/tests/e2e/replication/read_write_benchmark.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -138,7 +138,7 @@ int main(int argc, char **argv) { auto client = mg::e2e::replication::Connect(database_endpoint); client->Execute("SHOW INDEX INFO;"); if (const auto data = client->FetchAll()) { - if ((*data).size() != 0) { + if (!(*data).empty()) { LOG_FATAL("{} still have some indexes.", database_endpoint); } } else { diff --git a/tests/e2e/triggers/common.hpp b/tests/e2e/triggers/common.hpp index fbffd19f3..74593c1b9 100644 --- a/tests/e2e/triggers/common.hpp +++ b/tests/e2e/triggers/common.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -22,7 +22,7 @@ inline constexpr std::string_view kVertexLabel{"VERTEX"}; inline constexpr std::string_view kEdgeLabel{"EDGE"}; std::unique_ptr Connect(); -std::unique_ptr ConnectWithUser(const std::string_view username); +std::unique_ptr ConnectWithUser(std::string_view username); void CreateVertex(mg::Client &client, int vertex_id); void CreateEdge(mg::Client &client, int from_vertex, int to_vertex, int edge_id); diff --git a/tests/manual/antlr_tree_pretty_print.cpp b/tests/manual/antlr_tree_pretty_print.cpp index 0ba057f34..8957fc7e7 100644 --- a/tests/manual/antlr_tree_pretty_print.cpp +++ b/tests/manual/antlr_tree_pretty_print.cpp @@ -1,4 +1,4 @@ -// Copyright 2021 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -40,7 +40,7 @@ int main(int, const char **) { CommonTokenStream tokens(&lexer); tokens.fill(); - for (auto token : tokens.getTokens()) { + for (auto *token : tokens.getTokens()) { std::cout << token->toString() << std::endl; } diff --git a/tests/stress/long_running.cpp b/tests/stress/long_running.cpp index edf4b8452..37ed562ce 100644 --- a/tests/stress/long_running.cpp +++ b/tests/stress/long_running.cpp @@ -86,7 +86,7 @@ class GraphSession { std::set edges_; std::string indexed_label_; - std::set labels_; + std::set> labels_; std::map> labels_vertices_; @@ -109,7 +109,7 @@ class GraphSession { return *it; } - std::string RandomElement(std::set &data) { + std::string RandomElement(std::set> &data) { uint64_t pos = std::floor(GetRandom() * data.size()); auto it = data.begin(); std::advance(it, pos); diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index b7a90f22d..956cba781 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -6,7 +6,8 @@ find_package(Threads REQUIRED) add_custom_target(memgraph__unit) -set(memgraph_unit_main main.cpp) +add_library(memgraph_unit_main OBJECT main.cpp) +target_link_libraries(memgraph_unit_main mg-memory mg-utils gtest gmock Threads::Threads dl) function(add_unit_test test_cpp) _add_unit_test(${test_cpp} FALSE ${ARGN}) @@ -25,19 +26,19 @@ function(_add_unit_test test_cpp custom_main) ${test_cpp} ${ARGN}) - if(NOT ${custom_main}) - set(source_files - ${source_files} - ${memgraph_unit_main}) - endif() - add_executable(${target_name} ${source_files}) + if(NOT ${custom_main}) + target_link_libraries(${target_name} memgraph_unit_main) + else() + target_link_libraries(${target_name} gtest gmock Threads::Threads dl) + endif() + # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) - target_link_libraries(${target_name} mg-memory mg-utils gtest gmock Threads::Threads dl) + # register test if(TEST_COVERAGE) diff --git a/tests/unit/auth.cpp b/tests/unit/auth.cpp index 86a794480..6dbe20914 100644 --- a/tests/unit/auth.cpp +++ b/tests/unit/auth.cpp @@ -32,7 +32,7 @@ DECLARE_string(password_encryption_algorithm); class AuthWithStorage : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -40,7 +40,7 @@ class AuthWithStorage : public ::testing::Test { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; @@ -58,7 +58,7 @@ TEST_F(AuthWithStorage, RemoveRole) { ASSERT_TRUE(auth.RemoveRole("admin")); class AuthWithStorage : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -66,7 +66,7 @@ TEST_F(AuthWithStorage, RemoveRole) { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; @@ -589,9 +589,9 @@ TEST(AuthWithoutStorage, FineGrainedAccessPermissions) { } TEST_F(AuthWithStorage, FineGrainedAccessCheckerMerge) { - auto any_label = "AnyString"; - auto check_label = "Label"; - auto asterisk = "*"; + const auto *any_label = "AnyString"; + const auto *check_label = "Label"; + const auto *asterisk = "*"; { FineGrainedAccessPermissions fga_permissions1, fga_permissions2; @@ -929,7 +929,7 @@ TEST(AuthWithoutStorage, Crypto) { class AuthWithVariousEncryptionAlgorithms : public ::testing::Test { protected: - virtual void SetUp() { FLAGS_password_encryption_algorithm = "bcrypt"; } + void SetUp() override { FLAGS_password_encryption_algorithm = "bcrypt"; } }; TEST_F(AuthWithVariousEncryptionAlgorithms, VerifyPasswordDefault) { @@ -964,7 +964,7 @@ TEST_F(AuthWithVariousEncryptionAlgorithms, VerifyPasswordEmptyEncryptionThrow) class AuthWithStorageWithVariousEncryptionAlgorithms : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -973,7 +973,7 @@ class AuthWithStorageWithVariousEncryptionAlgorithms : public ::testing::Test { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; diff --git a/tests/unit/auth_handler.cpp b/tests/unit/auth_handler.cpp index e3352220f..6537575fd 100644 --- a/tests/unit/auth_handler.cpp +++ b/tests/unit/auth_handler.cpp @@ -36,12 +36,12 @@ class AuthQueryHandlerFixture : public testing::Test { #ifdef MG_ENTERPRISE memgraph::auth::FineGrainedAccessHandler handler{}; #endif - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { + void TearDown() override { std::filesystem::remove_all(test_folder_); perms = memgraph::auth::Permissions{}; #ifdef MG_ENTERPRISE diff --git a/tests/unit/bfs_common.hpp b/tests/unit/bfs_common.hpp index 3b16e4141..9bed67971 100644 --- a/tests/unit/bfs_common.hpp +++ b/tests/unit/bfs_common.hpp @@ -288,7 +288,7 @@ class Database { virtual std::pair, std::vector> BuildGraph(memgraph::query::DbAccessor *dba, const std::vector &vertex_locations, const std::vector> &edges) = 0; - virtual ~Database() {} + virtual ~Database() = default; void BfsTest(Database *db, int lower_bound, int upper_bound, memgraph::query::EdgeAtom::Direction direction, std::vector edge_types, bool known_sink, FilterLambdaType filter_lambda_type) { diff --git a/tests/unit/bolt_chunked_encoder_buffer.cpp b/tests/unit/bolt_chunked_encoder_buffer.cpp index 8dd23a0ff..47238b403 100644 --- a/tests/unit/bolt_chunked_encoder_buffer.cpp +++ b/tests/unit/bolt_chunked_encoder_buffer.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -87,7 +87,7 @@ TEST_F(BoltChunkedEncoderBuffer, TwoSmallChunks) { // the output array should look like this: // [0, 100, first 100 bytes of test data] + // [0, 100, second 100 bytes of test data] - auto data = output_stream.output.data(); + auto *data = output_stream.output.data(); VerifyChunkOfTestData(data, size1); VerifyChunkOfTestData(data + kChunkHeaderSize + size1, size2, size1); } @@ -105,7 +105,7 @@ TEST_F(BoltChunkedEncoderBuffer, OneAndAHalfOfMaxChunk) { // the output array should look like this: // [0xFF, 0xFF, first 65535 bytes of test data, // 0x86, 0xA1, 34465 bytes of test data after the first 65535 bytes] - auto output = output_stream.output.data(); + auto *output = output_stream.output.data(); VerifyChunkOfTestData(output, kChunkMaxDataSize); VerifyChunkOfTestData(output + kChunkWholeSize, kTestDataSize - kChunkMaxDataSize, kChunkMaxDataSize); } diff --git a/tests/unit/cpp_api.cpp b/tests/unit/cpp_api.cpp index ecad43487..929a70431 100644 --- a/tests/unit/cpp_api.cpp +++ b/tests/unit/cpp_api.cpp @@ -27,7 +27,7 @@ template struct CppApiTestFixture : public ::testing::Test { protected: - virtual void SetUp() override { mgp::mrd.Register(&memory); } + void SetUp() override { mgp::mrd.Register(&memory); } void TearDown() override { if (std::is_same::value) { diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index 1d7f236a2..54453de09 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -58,7 +58,7 @@ class Base { ParsingContext context_; Parameters parameters_; - virtual ~Base() {} + virtual ~Base() = default; virtual Query *ParseQuery(const std::string &query_string) = 0; @@ -199,8 +199,8 @@ class CachedAstGenerator : public Base { class MockModule : public procedure::Module { public: - MockModule(){}; - ~MockModule() override{}; + MockModule() = default; + ~MockModule() override = default; MockModule(const MockModule &) = delete; MockModule(MockModule &&) = delete; MockModule &operator=(const MockModule &) = delete; diff --git a/tests/unit/integrations_kafka_consumer.cpp b/tests/unit/integrations_kafka_consumer.cpp index 63a419552..3d5feb80b 100644 --- a/tests/unit/integrations_kafka_consumer.cpp +++ b/tests/unit/integrations_kafka_consumer.cpp @@ -48,7 +48,7 @@ inline constexpr int64_t kDefaultBatchSize{1000}; } // namespace struct ConsumerTest : public ::testing::Test { - ConsumerTest() {} + ConsumerTest() = default; ConsumerInfo CreateDefaultConsumerInfo() const { const auto test_name = std::string{::testing::UnitTest::GetInstance()->current_test_info()->name()}; @@ -132,7 +132,7 @@ TEST_F(ConsumerTest, BatchInterval) { info.batch_interval = kBatchInterval; auto expected_messages_received = true; auto consumer_function = [&](const std::vector &messages) mutable { - received_timestamps.push_back({messages.size(), std::chrono::steady_clock::now()}); + received_timestamps.emplace_back(messages.size(), std::chrono::steady_clock::now()); for (const auto &message : messages) { expected_messages_received &= (kMessage == std::string_view(message.Payload().data(), message.Payload().size())); } @@ -227,7 +227,7 @@ TEST_F(ConsumerTest, BatchSize) { static constexpr std::string_view kMessage = "BatchSizeTestMessage"; auto expected_messages_received = true; auto consumer_function = [&](const std::vector &messages) mutable { - received_timestamps.push_back({messages.size(), std::chrono::steady_clock::now()}); + received_timestamps.emplace_back(messages.size(), std::chrono::steady_clock::now()); for (const auto &message : messages) { expected_messages_received &= (kMessage == std::string_view(message.Payload().data(), message.Payload().size())); } diff --git a/tests/unit/interpreter.cpp b/tests/unit/interpreter.cpp index 5eb7cd539..57bb79db8 100644 --- a/tests/unit/interpreter.cpp +++ b/tests/unit/interpreter.cpp @@ -64,7 +64,7 @@ class InterpreterTest : public ::testing::Test { const std::string testSuiteCsv = "interpreter_csv"; std::filesystem::path data_directory = std::filesystem::temp_directory_path() / "MG_tests_unit_interpreter"; - InterpreterTest() {} + InterpreterTest() = default; memgraph::storage::Config config{ [&]() { @@ -334,7 +334,7 @@ TYPED_TEST(InterpreterTest, Bfs) { auto kNumUnreachableNodes = 1000; auto kNumUnreachableEdges = 100000; auto kResCoeff = 5; - const auto kReachable = "reachable"; + const auto *const kReachable = "reachable"; const auto kId = "id"; if (std::is_same::value) { diff --git a/tests/unit/kvstore.cpp b/tests/unit/kvstore.cpp index 8af05e482..48ab7dc82 100644 --- a/tests/unit/kvstore.cpp +++ b/tests/unit/kvstore.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -20,9 +20,9 @@ namespace fs = std::filesystem; class KVStore : public ::testing::Test { protected: - virtual void SetUp() { memgraph::utils::EnsureDir(test_folder_); } + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / ("unit_kvstore_test_" + std::to_string(static_cast(getpid())))}; diff --git a/tests/unit/mgp_kafka_c_api.cpp b/tests/unit/mgp_kafka_c_api.cpp index 91218dc75..828d52163 100644 --- a/tests/unit/mgp_kafka_c_api.cpp +++ b/tests/unit/mgp_kafka_c_api.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -113,7 +113,7 @@ class MgpApiTest : public ::testing::Test { using KafkaMessage = MockedRdKafkaMessage; MgpApiTest() { messages_.emplace(CreateMockedBatch()); } - ~MgpApiTest() { messages_.reset(); } + ~MgpApiTest() override { messages_.reset(); } mgp_messages &Messages() { return *messages_; } diff --git a/tests/unit/plan_pretty_print.cpp b/tests/unit/plan_pretty_print.cpp index 94b7342f9..97f1355cb 100644 --- a/tests/unit/plan_pretty_print.cpp +++ b/tests/unit/plan_pretty_print.cpp @@ -46,7 +46,7 @@ class PrintToJsonTest : public ::testing::Test { dba_storage(db->Access()), dba(dba_storage.get()) {} - ~PrintToJsonTest() { + ~PrintToJsonTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index e06d63908..a14ef2d30 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -46,9 +46,7 @@ #include "storage/v2/id_types.hpp" #include "utils/string.hpp" -namespace memgraph::query { - -namespace test_common { +namespace memgraph::query::test_common { auto ToIntList(const TypedValue &t) { std::vector list; @@ -189,7 +187,7 @@ auto GetEdgeVariable(AstStorage &storage, const std::string &name, EdgeAtom::Typ for (const auto &type : edge_types) { types.push_back(storage.GetEdgeTypeIx(type)); } - auto r_val = storage.Create(storage.Create(name), type, dir, types); + auto *r_val = storage.Create(storage.Create(name), type, dir, types); r_val->filter_lambda_.inner_edge = flambda_inner_edge ? flambda_inner_edge : storage.Create(memgraph::utils::RandomString(20)); @@ -215,14 +213,14 @@ auto GetEdgeVariable(AstStorage &storage, const std::string &name, EdgeAtom::Typ /// Name is used to create the Identifier which is assigned to the node. auto GetNode(AstStorage &storage, const std::string &name, std::optional label = std::nullopt, const bool user_declared = true) { - auto node = storage.Create(storage.Create(name, user_declared)); + auto *node = storage.Create(storage.Create(name, user_declared)); if (label) node->labels_.emplace_back(storage.GetLabelIx(*label)); return node; } /// Create a Pattern with given atoms. auto GetPattern(AstStorage &storage, std::vector atoms) { - auto pattern = storage.Create(); + auto *pattern = storage.Create(); pattern->identifier_ = storage.Create(memgraph::utils::RandomString(20), false); pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); return pattern; @@ -230,7 +228,7 @@ auto GetPattern(AstStorage &storage, std::vector atoms) { /// Create a Pattern with given name and atoms. auto GetPattern(AstStorage &storage, const std::string &name, std::vector atoms) { - auto pattern = storage.Create(); + auto *pattern = storage.Create(); pattern->identifier_ = storage.Create(name, true); pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); return pattern; @@ -377,7 +375,7 @@ void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &na /// @sa GetWith template auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { - auto ret = storage.Create(); + auto *ret = storage.Create(); ret->body_.distinct = distinct; FillReturnBody(storage, ret->body_, exprs...); return ret; @@ -390,7 +388,7 @@ auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { /// @sa GetReturn template auto GetWith(AstStorage &storage, bool distinct, T... exprs) { - auto with = storage.Create(); + auto *with = storage.Create(); with->body_.distinct = distinct; FillReturnBody(storage, with->body_, exprs...); return with; @@ -407,7 +405,7 @@ auto GetUnwind(AstStorage &storage, Expression *expr, NamedExpression *as) { /// Create the delete clause with given named expressions. auto GetDelete(AstStorage &storage, std::vector exprs, bool detach = false) { - auto del = storage.Create(); + auto *del = storage.Create(); del->expressions_.insert(del->expressions_.begin(), exprs.begin(), exprs.end()); del->detach_ = detach; return del; @@ -495,9 +493,7 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec return storage.Create(named_expr, clauses); } -} // namespace test_common - -} // namespace memgraph::query +} // namespace memgraph::query::test_common /// All the following macros implicitly pass `storage` variable to functions. /// You need to have `AstStorage storage;` somewhere in scope to use them. diff --git a/tests/unit/query_cost_estimator.cpp b/tests/unit/query_cost_estimator.cpp index a2c7b4a48..f3d6b3864 100644 --- a/tests/unit/query_cost_estimator.cpp +++ b/tests/unit/query_cost_estimator.cpp @@ -49,7 +49,7 @@ class QueryCostEstimator : public ::testing::Test { Parameters parameters_; int symbol_count = 0; - void SetUp() { + void SetUp() override { { auto unique_acc = db->UniqueAccess(); ASSERT_FALSE(unique_acc->CreateIndex(label).HasError()); @@ -131,7 +131,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelCardinality) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", const_val); EXPECT_COST(1 * CostParam::MakeScanAllByLabelPropertyValue); } @@ -139,7 +139,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstExpr) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", // once we make expression const-folding this test case will fail storage_.Create(const_val)); @@ -149,7 +149,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstExpr) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeUpperConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", nullopt, InclusiveBound(const_val)); // cardinality estimation is exact for very small indexes @@ -159,7 +159,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeUpperConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeLowerConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(17), Parameter(17)}) { + for (auto *const_val : {Literal(17), Parameter(17)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", InclusiveBound(const_val), nullopt); // cardinality estimation is exact for very small indexes @@ -169,7 +169,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeLowerConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeConstExpr) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { auto bound = std::make_optional( memgraph::utils::MakeBoundInclusive(static_cast(storage_.Create(const_val)))); MakeOp(nullptr, NextSymbol(), label, property, "property", bound, nullopt); diff --git a/tests/unit/query_dump.cpp b/tests/unit/query_dump.cpp index f212d9189..5556ab90a 100644 --- a/tests/unit/query_dump.cpp +++ b/tests/unit/query_dump.cpp @@ -46,7 +46,7 @@ const char *kRemoveInternalLabelProperty = "MATCH (u) REMOVE u:__mg_vertex__, u. struct DatabaseState { struct Vertex { int64_t id; - std::set labels; + std::set> labels; std::map props; }; @@ -67,7 +67,7 @@ struct DatabaseState { struct LabelPropertiesItem { std::string label; - std::set properties; + std::set> properties; }; std::set vertices; @@ -139,7 +139,7 @@ DatabaseState GetState(memgraph::storage::Storage *db) { std::set vertices; auto dba = db->Access(); for (const auto &vertex : dba->Vertices(memgraph::storage::View::NEW)) { - std::set labels; + std::set> labels; auto maybe_labels = vertex.Labels(memgraph::storage::View::NEW); MG_ASSERT(maybe_labels.HasValue()); for (const auto &label : *maybe_labels) { @@ -198,7 +198,7 @@ DatabaseState GetState(memgraph::storage::Storage *db) { existence_constraints.insert({dba->LabelToName(item.first), dba->PropertyToName(item.second)}); } for (const auto &item : info.unique) { - std::set properties; + std::set> properties; for (const auto &property : item.second) { properties.insert(dba->PropertyToName(property)); } diff --git a/tests/unit/query_expression_evaluator.cpp b/tests/unit/query_expression_evaluator.cpp index 86ac5d624..44d3ed301 100644 --- a/tests/unit/query_expression_evaluator.cpp +++ b/tests/unit/query_expression_evaluator.cpp @@ -69,7 +69,7 @@ class ExpressionEvaluatorTest : public ::testing::Test { storage_dba(db->Access()), dba(storage_dba.get()) {} - ~ExpressionEvaluatorTest() { + ~ExpressionEvaluatorTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } @@ -580,7 +580,7 @@ TYPED_TEST(ExpressionEvaluatorTest, VertexAndEdgeIndexing) { TYPED_TEST(ExpressionEvaluatorTest, TypedValueListIndexing) { auto list_vector = memgraph::utils::pmr::vector(this->ctx.memory); list_vector.emplace_back("string1"); - list_vector.emplace_back(TypedValue("string2")); + list_vector.emplace_back("string2"); auto *identifier = this->storage.template Create("n"); auto node_symbol = this->symbol_table.CreateSymbol("n", true); @@ -1196,7 +1196,7 @@ class ExpressionEvaluatorPropertyLookup : public ExpressionEvaluatorTeststorage.template Create("element"); Symbol symbol = this->symbol_table.CreateSymbol("element", true); - void SetUp() { identifier->MapTo(symbol); } + void SetUp() override { identifier->MapTo(symbol); } auto Value(std::pair property) { auto *op = this->storage.template Create(identifier, this->storage.GetPropertyIx(property.first)); @@ -1388,7 +1388,7 @@ class ExpressionEvaluatorAllPropertiesLookup : public ExpressionEvaluatorTeststorage.template Create("element"); Symbol symbol = this->symbol_table.CreateSymbol("element", true); - void SetUp() { identifier->MapTo(symbol); } + void SetUp() override { identifier->MapTo(symbol); } auto Value() { auto *op = this->storage.template Create(identifier); diff --git a/tests/unit/query_plan_checker.hpp b/tests/unit/query_plan_checker.hpp index 37695b581..6f2f23df7 100644 --- a/tests/unit/query_plan_checker.hpp +++ b/tests/unit/query_plan_checker.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "query/frontend/semantic/symbol_generator.hpp" #include "query/frontend/semantic/symbol_table.hpp" @@ -23,7 +24,7 @@ namespace memgraph::query::plan { class BaseOpChecker { public: - virtual ~BaseOpChecker() {} + virtual ~BaseOpChecker() = default; virtual void CheckOp(LogicalOperator &, const SymbolTable &) = 0; }; @@ -187,7 +188,7 @@ using ExpectEvaluatePatternFilter = OpChecker; class ExpectFilter : public OpChecker { public: - ExpectFilter(const std::vector> &pattern_filters = {}) + explicit ExpectFilter(const std::vector> &pattern_filters = {}) : pattern_filters_(pattern_filters) {} void ExpectOp(Filter &filter, const SymbolTable &symbol_table) override { @@ -222,7 +223,7 @@ class ExpectForeach : public OpChecker { class ExpectApply : public OpChecker { public: - ExpectApply(const std::list &subquery) : subquery_(subquery) {} + explicit ExpectApply(const std::list &subquery) : subquery_(subquery) {} void ExpectOp(Apply &apply, const SymbolTable &symbol_table) override { PlanChecker check_subquery(subquery_, symbol_table); @@ -287,7 +288,7 @@ class ExpectAggregate : public OpChecker { auto aggr_it = aggregations_.begin(); for (const auto &aggr_elem : op.aggregations_) { ASSERT_NE(aggr_it, aggregations_.end()); - auto aggr = *aggr_it++; + auto *aggr = *aggr_it++; // TODO: Proper expression equality EXPECT_EQ(typeid(aggr_elem.value).hash_code(), typeid(aggr->expression1_).hash_code()); EXPECT_EQ(typeid(aggr_elem.key).hash_code(), typeid(aggr->expression2_).hash_code()); @@ -472,9 +473,9 @@ class ExpectIndexedJoin : public OpChecker { class ExpectCallProcedure : public OpChecker { public: - ExpectCallProcedure(const std::string &name, const std::vector &args, + ExpectCallProcedure(std::string name, const std::vector &args, const std::vector &fields, const std::vector &result_syms) - : name_(name), args_(args), fields_(fields), result_syms_(result_syms) {} + : name_(std::move(name)), args_(args), fields_(fields), result_syms_(result_syms) {} void ExpectOp(CallProcedure &op, const SymbolTable &symbol_table) override { EXPECT_EQ(op.procedure_name_, name_); @@ -526,7 +527,7 @@ class FakeDbAccessor { } int64_t VerticesCount(memgraph::storage::LabelId label, memgraph::storage::PropertyId property) const { - for (auto &index : label_property_index_) { + for (const auto &index : label_property_index_) { if (std::get<0>(index) == label && std::get<1>(index) == property) { return std::get<2>(index); } @@ -539,7 +540,7 @@ class FakeDbAccessor { } bool LabelPropertyIndexExists(memgraph::storage::LabelId label, memgraph::storage::PropertyId property) const { - for (auto &index : label_property_index_) { + for (const auto &index : label_property_index_) { if (std::get<0>(index) == label && std::get<1>(index) == property) { return true; } diff --git a/tests/unit/query_plan_common.hpp b/tests/unit/query_plan_common.hpp index 4deba7648..1831a53d7 100644 --- a/tests/unit/query_plan_common.hpp +++ b/tests/unit/query_plan_common.hpp @@ -64,7 +64,7 @@ std::vector> CollectProduce(const Produce &produce, Exec // collect the symbols from the return clause std::vector symbols; - for (auto named_expression : produce.named_expressions_) + for (auto *named_expression : produce.named_expressions_) symbols.emplace_back(context->symbol_table.at(*named_expression)); // stream out results @@ -109,7 +109,7 @@ struct ScanAllTuple { ScanAllTuple MakeScanAll(AstStorage &storage, SymbolTable &symbol_table, const std::string &identifier, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, view); @@ -125,7 +125,7 @@ ScanAllTuple MakeScanAll(AstStorage &storage, SymbolTable &symbol_table, const s ScanAllTuple MakeScanAllByLabel(AstStorage &storage, SymbolTable &symbol_table, const std::string &identifier, memgraph::storage::LabelId label, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, label, view); @@ -144,7 +144,7 @@ ScanAllTuple MakeScanAllByLabelPropertyRange(AstStorage &storage, SymbolTable &s std::optional upper_bound, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, label, property, property_name, @@ -163,7 +163,7 @@ ScanAllTuple MakeScanAllByLabelPropertyValue(AstStorage &storage, SymbolTable &s const std::string &property_name, Expression *value, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = @@ -183,11 +183,11 @@ ExpandTuple MakeExpand(AstStorage &storage, SymbolTable &symbol_table, std::shar Symbol input_symbol, const std::string &edge_identifier, EdgeAtom::Direction direction, const std::vector &edge_types, const std::string &node_identifier, bool existing_node, memgraph::storage::View view) { - auto edge = memgraph::query::test_common::GetEdge(storage, edge_identifier, direction); + auto *edge = memgraph::query::test_common::GetEdge(storage, edge_identifier, direction); auto edge_sym = symbol_table.CreateSymbol(edge_identifier, true); edge->identifier_->MapTo(edge_sym); - auto node = memgraph::query::test_common::GetNode(storage, node_identifier); + auto *node = memgraph::query::test_common::GetNode(storage, node_identifier); auto node_sym = symbol_table.CreateSymbol(node_identifier, true); node->identifier_->MapTo(node_sym); diff --git a/tests/unit/query_plan_edge_cases.cpp b/tests/unit/query_plan_edge_cases.cpp index 52769fa55..d0953651e 100644 --- a/tests/unit/query_plan_edge_cases.cpp +++ b/tests/unit/query_plan_edge_cases.cpp @@ -43,7 +43,7 @@ class QueryExecution : public testing::Test { std::optional repl_state; std::optional> db_gk; - void SetUp() { + void SetUp() override { auto config = [&]() { memgraph::storage::Config config{}; config.durability.storage_directory = data_directory; @@ -70,7 +70,7 @@ class QueryExecution : public testing::Test { interpreter_.emplace(&*interpreter_context_, *db_acc_); } - void TearDown() { + void TearDown() override { interpreter_ = std::nullopt; interpreter_context_ = std::nullopt; db_acc_.reset(); diff --git a/tests/unit/query_plan_operator_to_string.cpp b/tests/unit/query_plan_operator_to_string.cpp index 36918425d..27d27f0d1 100644 --- a/tests/unit/query_plan_operator_to_string.cpp +++ b/tests/unit/query_plan_operator_to_string.cpp @@ -40,7 +40,7 @@ class OperatorToStringTest : public ::testing::Test { dba_storage(db->Access()), dba(dba_storage.get()) {} - ~OperatorToStringTest() { + ~OperatorToStringTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } diff --git a/tests/unit/query_required_privileges.cpp b/tests/unit/query_required_privileges.cpp index e43525f5b..630bc44a0 100644 --- a/tests/unit/query_required_privileges.cpp +++ b/tests/unit/query_required_privileges.cpp @@ -153,7 +153,7 @@ TEST_F(TestPrivilegeExtractor, DumpDatabase) { } TEST_F(TestPrivilegeExtractor, ReadFile) { - auto load_csv = storage.Create(); + auto *load_csv = storage.Create(); load_csv->row_var_ = IDENT("row"); auto *query = QUERY(SINGLE_QUERY(load_csv)); EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::READ_FILE)); diff --git a/tests/unit/query_semantic.cpp b/tests/unit/query_semantic.cpp index d49eb3798..5d015f103 100644 --- a/tests/unit/query_semantic.cpp +++ b/tests/unit/query_semantic.cpp @@ -54,17 +54,17 @@ TYPED_TEST(TestSymbolGenerator, MatchNodeReturn) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for pattern, node_atom_1 and named_expr in return EXPECT_EQ(symbol_table.max_position(), 3); - auto match = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = match->patterns_[0]; + auto *match = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = match->patterns_[0]; auto pattern_sym = symbol_table.at(*pattern->identifier_); EXPECT_EQ(pattern_sym.type(), Symbol::Type::PATH); EXPECT_FALSE(pattern_sym.user_declared()); - auto node_atom = dynamic_cast(pattern->atoms_[0]); + auto *node_atom = dynamic_cast(pattern->atoms_[0]); auto node_sym = symbol_table.at(*node_atom->identifier_); EXPECT_EQ(node_sym.name(), "node_atom_1"); EXPECT_EQ(node_sym.type(), Symbol::Type::VERTEX); - auto ret = dynamic_cast(query_ast->single_query_->clauses_[1]); - auto named_expr = ret->body_.named_expressions[0]; + auto *ret = dynamic_cast(query_ast->single_query_->clauses_[1]); + auto *named_expr = ret->body_.named_expressions[0]; auto column_sym = symbol_table.at(*named_expr); EXPECT_EQ(node_sym.name(), column_sym.name()); EXPECT_NE(node_sym, column_sym); @@ -78,8 +78,8 @@ TYPED_TEST(TestSymbolGenerator, MatchNamedPattern) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for p, node_atom_1 and named_expr in return EXPECT_EQ(symbol_table.max_position(), 3); - auto match = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = match->patterns_[0]; + auto *match = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = match->patterns_[0]; auto pattern_sym = symbol_table.at(*pattern->identifier_); EXPECT_EQ(pattern_sym.type(), Symbol::Type::PATH); EXPECT_EQ(pattern_sym.name(), "p"); @@ -114,14 +114,14 @@ TYPED_TEST(TestSymbolGenerator, CreateNodeReturn) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for pattern, `n` and named_expr EXPECT_EQ(symbol_table.max_position(), 3); - auto create = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = create->patterns_[0]; - auto node_atom = dynamic_cast(pattern->atoms_[0]); + auto *create = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = create->patterns_[0]; + auto *node_atom = dynamic_cast(pattern->atoms_[0]); auto node_sym = symbol_table.at(*node_atom->identifier_); EXPECT_EQ(node_sym.name(), "n"); EXPECT_EQ(node_sym.type(), Symbol::Type::VERTEX); - auto ret = dynamic_cast(query_ast->single_query_->clauses_[1]); - auto named_expr = ret->body_.named_expressions[0]; + auto *ret = dynamic_cast(query_ast->single_query_->clauses_[1]); + auto *named_expr = ret->body_.named_expressions[0]; auto column_sym = symbol_table.at(*named_expr); EXPECT_EQ(node_sym.name(), column_sym.name()); EXPECT_NE(node_sym, column_sym); @@ -151,7 +151,7 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateRedeclareNode) { TYPED_TEST(TestSymbolGenerator, MatchCreateRedeclareEdge) { // AST with redeclaring a match edge variable in create: // MATCH (n) -[r]- (m) CREATE (n) -[r :relationship]-> (l) - auto relationship = "relationship"; + const auto *relationship = "relationship"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::OUT, {relationship}), NODE("l"))))); @@ -176,8 +176,8 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateTypeMismatch) { TYPED_TEST(TestSymbolGenerator, CreateMultipleEdgeType) { // Multiple edge relationship are not allowed when creating edges. // CREATE (n) -[r :rel1 | :rel2]-> (m) - auto rel1 = "rel1"; - auto rel2 = "rel2"; + const auto *rel1 = "rel1"; + const auto *rel2 = "rel2"; auto edge = EDGE("r", EdgeAtom::Direction::OUT, {rel1}); edge->edge_types_.emplace_back(this->storage.GetEdgeTypeIx(rel2)); auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), edge, NODE("m"))))); @@ -187,7 +187,7 @@ TYPED_TEST(TestSymbolGenerator, CreateMultipleEdgeType) { TYPED_TEST(TestSymbolGenerator, CreateBidirectionalEdge) { // Bidirectional relationships are not allowed when creating edges. // CREATE (n) -[r :rel1]- (m) - auto rel1 = "rel1"; + const auto *rel1 = "rel1"; auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::BOTH, {rel1}), NODE("m"))))); EXPECT_THROW(memgraph::query::MakeSymbolTable(query), SemanticException); } @@ -276,8 +276,8 @@ TYPED_TEST(TestSymbolGenerator, MatchWithWhereUnbound) { TYPED_TEST(TestSymbolGenerator, CreateMultiExpand) { // Test CREATE (n) -[r :r]-> (m), (n) - [p :p]-> (l) - auto r_type = "r"; - auto p_type = "p"; + const auto *r_type = "r"; + const auto *p_type = "p"; auto node_n1 = NODE("n"); auto edge_r = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); auto node_m = NODE("m"); @@ -308,8 +308,8 @@ TYPED_TEST(TestSymbolGenerator, CreateMultiExpand) { TYPED_TEST(TestSymbolGenerator, MatchCreateExpandLabel) { // Test MATCH (n) CREATE (m) -[r :r]-> (n:label) - auto r_type = "r"; - auto label = "label"; + const auto *r_type = "r"; + const auto *label = "label"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), CREATE(PATTERN(NODE("m"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), NODE("n", label))))); @@ -318,7 +318,7 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateExpandLabel) { TYPED_TEST(TestSymbolGenerator, CreateExpandProperty) { // Test CREATE (n) -[r :r]-> (n {prop: 42}) - auto r_type = "r"; + const auto *r_type = "r"; auto n_prop = NODE("n"); std::get<0>(n_prop->properties_)[this->storage.GetPropertyIx("prop")] = LITERAL(42); auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), n_prop)))); @@ -379,7 +379,7 @@ TYPED_TEST(TestSymbolGenerator, MatchPropCreateNodeProp) { TYPED_TEST(TestSymbolGenerator, CreateNodeEdge) { // Test CREATE (n), (n) -[r :r]-> (n) - auto r_type = "r"; + const auto *r_type = "r"; auto node_1 = NODE("n"); auto node_2 = NODE("n"); auto edge = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); @@ -396,7 +396,7 @@ TYPED_TEST(TestSymbolGenerator, CreateNodeEdge) { TYPED_TEST(TestSymbolGenerator, MatchWithCreate) { // Test MATCH (n) WITH n AS m CREATE (m) -[r :r]-> (m) - auto r_type = "r"; + const auto *r_type = "r"; auto node_1 = NODE("n"); auto node_2 = NODE("m"); auto edge = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); @@ -500,7 +500,7 @@ TYPED_TEST(TestSymbolGenerator, MergeVariableError) { TYPED_TEST(TestSymbolGenerator, MergeVariableErrorEdge) { // Test MATCH (n) -[r]- (m) MERGE (a) -[r :rel]- (b) - auto rel = "rel"; + const auto *rel = "rel"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), MERGE(PATTERN(NODE("a"), EDGE("r", EdgeAtom::Direction::BOTH, {rel}), NODE("b"))))); EXPECT_THROW(memgraph::query::MakeSymbolTable(query), RedeclareVariableError); @@ -516,7 +516,7 @@ TYPED_TEST(TestSymbolGenerator, MergeEdgeWithoutType) { TYPED_TEST(TestSymbolGenerator, MergeOnMatchOnCreate) { // Test MATCH (n) MERGE (n) -[r :rel]- (m) ON MATCH SET n.prop = 42 // ON CREATE SET m.prop = 42 RETURN r AS r - auto rel = "rel"; + const auto *rel = "rel"; auto prop = this->dba.NameToProperty("prop"); auto match_n = NODE("n"); auto merge_n = NODE("n"); @@ -641,8 +641,8 @@ TYPED_TEST(TestSymbolGenerator, MatchReturnAsteriskNoUserVariables) { TYPED_TEST(TestSymbolGenerator, MatchMergeExpandLabel) { // Test MATCH (n) MERGE (m) -[r :r]-> (n:label) - auto r_type = "r"; - auto label = "label"; + const auto *r_type = "r"; + const auto *label = "label"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), MERGE(PATTERN(NODE("m"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), NODE("n", label))))); diff --git a/tests/unit/rpc_messages.hpp b/tests/unit/rpc_messages.hpp index 6058c37cf..2f8175a36 100644 --- a/tests/unit/rpc_messages.hpp +++ b/tests/unit/rpc_messages.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "rpc/messages.hpp" #include "slk/serialization.hpp" #include "utils/typeinfo.hpp" @@ -18,7 +20,7 @@ struct SumReq { static const memgraph::utils::TypeInfo kType; - SumReq() {} // Needed for serialization. + SumReq() = default; // Needed for serialization. SumReq(int x, int y) : x(x), y(y) {} static void Load(SumReq *obj, memgraph::slk::Reader *reader); @@ -33,8 +35,8 @@ const memgraph::utils::TypeInfo SumReq::kType{memgraph::utils::TypeId::UNKNOWN, struct SumRes { static const memgraph::utils::TypeInfo kType; - SumRes() {} // Needed for serialization. - SumRes(int sum) : sum(sum) {} + SumRes() = default; // Needed for serialization. + explicit SumRes(int sum) : sum(sum) {} static void Load(SumRes *obj, memgraph::slk::Reader *reader); static void Save(const SumRes &obj, memgraph::slk::Builder *builder); @@ -57,8 +59,8 @@ using Sum = memgraph::rpc::RequestResponse; struct EchoMessage { static const memgraph::utils::TypeInfo kType; - EchoMessage() {} // Needed for serialization. - EchoMessage(const std::string &data) : data(data) {} + EchoMessage() = default; // Needed for serialization. + explicit EchoMessage(std::string data) : data(std::move(data)) {} static void Load(EchoMessage *obj, memgraph::slk::Reader *reader); static void Save(const EchoMessage &obj, memgraph::slk::Builder *builder); diff --git a/tests/unit/slk_core.cpp b/tests/unit/slk_core.cpp index 65b916db3..9b63b82f8 100644 --- a/tests/unit/slk_core.cpp +++ b/tests/unit/slk_core.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -108,7 +108,7 @@ TEST(SlkCore, SetPrimitive) { } TEST(SlkCore, SetString) { - std::set original{"hai hai hai", "nandare!"}; + std::set> original{"hai hai hai", "nandare!"}; memgraph::slk::Loopback loopback; auto builder = loopback.GetBuilder(); memgraph::slk::Save(original, builder); @@ -116,7 +116,7 @@ TEST(SlkCore, SetString) { for (const auto &item : original) { size += sizeof(uint64_t) + item.size(); } - std::set decoded; + std::set> decoded; auto reader = loopback.GetReader(); memgraph::slk::Load(&decoded, reader); ASSERT_EQ(original, decoded); diff --git a/tests/unit/slk_streams.cpp b/tests/unit/slk_streams.cpp index 634eaecb5..8a673af1c 100644 --- a/tests/unit/slk_streams.cpp +++ b/tests/unit/slk_streams.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -66,7 +66,7 @@ std::vector BufferToBinaryData(const uint8_t *data, size_t size, std size_t pos = 0; for (size_t i = 0; i < sizes.size(); ++i) { EXPECT_GE(size, pos + sizes[i]); - ret.push_back({data + pos, sizes[i]}); + ret.emplace_back(data + pos, sizes[i]); pos += sizes[i]; } return ret; diff --git a/tests/unit/storage_rocks.cpp b/tests/unit/storage_rocks.cpp index 42890c383..6d5db7d75 100644 --- a/tests/unit/storage_rocks.cpp +++ b/tests/unit/storage_rocks.cpp @@ -49,7 +49,7 @@ class RocksDBStorageTest : public ::testing::TestWithParam { disk_test_utils::RemoveRocksDbDirs(testSuite); } - ~RocksDBStorageTest() override {} + ~RocksDBStorageTest() override = default; protected: std::unique_ptr storage; diff --git a/tests/unit/storage_v2_property_store.cpp b/tests/unit/storage_v2_property_store.cpp index 9da503f71..59b38c632 100644 --- a/tests/unit/storage_v2_property_store.cpp +++ b/tests/unit/storage_v2_property_store.cpp @@ -670,7 +670,7 @@ TEST(PropertyStore, SetMultipleProperties) { const std::map data_in_map{data.begin(), data.end()}; auto check_store = [data](const memgraph::storage::PropertyStore &store) { - for (auto &[key, value] : data) { + for (const auto &[key, value] : data) { ASSERT_TRUE(store.IsPropertyEqual(key, value)); } }; diff --git a/tests/unit/storage_v2_wal_file.cpp b/tests/unit/storage_v2_wal_file.cpp index 19b633f9f..a67b09305 100644 --- a/tests/unit/storage_v2_wal_file.cpp +++ b/tests/unit/storage_v2_wal_file.cpp @@ -231,7 +231,7 @@ class DeltaGenerator final { } void AppendOperation(memgraph::storage::durability::StorageMetadataOperation operation, const std::string &label, - const std::set properties = {}, const std::string &stats = {}) { + const std::set> properties = {}, const std::string &stats = {}) { auto label_id = memgraph::storage::LabelId::FromUint(mapper_.NameToId(label)); std::set property_ids; for (const auto &property : properties) { @@ -378,7 +378,7 @@ void AssertWalDataEqual(const DeltaGenerator::DataT &data, const std::filesystem class WalFileTest : public ::testing::TestWithParam { public: - WalFileTest() {} + WalFileTest() = default; void SetUp() override { Clear(); } @@ -710,7 +710,7 @@ TEST_P(WalFileTest, PartialData) { class StorageModeWalFileTest : public ::testing::TestWithParam { public: - StorageModeWalFileTest() {} + StorageModeWalFileTest() = default; void SetUp() override { Clear(); } diff --git a/tests/unit/utils_file.cpp b/tests/unit/utils_file.cpp index 05d8184be..89c3699a9 100644 --- a/tests/unit/utils_file.cpp +++ b/tests/unit/utils_file.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -132,7 +132,7 @@ class UtilsFileTest : public ::testing::Test { private: void Clear() { if (fs::exists(storage)) { - for (auto &file : fs::recursive_directory_iterator(storage)) { + for (const auto &file : fs::recursive_directory_iterator(storage)) { std::error_code error_code; // For exception suppression. fs::permissions(file.path(), fs::perms::owner_all, fs::perm_options::add, error_code); } diff --git a/tests/unit/utils_string.cpp b/tests/unit/utils_string.cpp index 4cde1dfe1..cefe57a6a 100644 --- a/tests/unit/utils_string.cpp +++ b/tests/unit/utils_string.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -151,7 +151,7 @@ TEST(String, RandomString) { EXPECT_EQ(RandomString(1).size(), 1); EXPECT_EQ(RandomString(42).size(), 42); - std::set string_set; + std::set> string_set; for (int i = 0; i < 20; ++i) string_set.emplace(RandomString(256)); EXPECT_EQ(string_set.size(), 20); From 8b9e1fa08bd2094bf89313e94fe31028d28f4ff1 Mon Sep 17 00:00:00 2001 From: andrejtonev <29177572+andrejtonev@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:02:35 +0100 Subject: [PATCH 25/52] Replication refactor part 6 (#1484) Single (instance level) connection to a replica (messages from all databases get multiplexed through it) ReplicationClient split in two: ReplicationClient and ReplicationStorageClient New ReplicationClient, moved under replication, handles the raw connection, owned by MainRoleData ReplicationStorageClient handles the storage <-> replica state machine and holds to a stream Removed epoch and storage from *Clients rpc::Stream proactively aborts on error and sets itself to a defunct state Removed HandleRpcFailure, instead we simply log the error and let the FrequentCheck handle re-connection replica_state is now a synced variable ReplicaStorageClient state machine bugfixes Single FrequentCheck that goes through DBMS Moved ReplicationState under DbmsHandler Moved some replication startup logic under the DbmsHandler's constructor Removed InMemoryReplicationClient CreateReplicationClient has been removed from Storage Simplified GetRecoverySteps and made safer --------- Co-authored-by: Gareth Lloyd --- release/get_version.py | 14 +- src/dbms/CMakeLists.txt | 2 +- src/dbms/constants.hpp | 6 + src/dbms/database.cpp | 2 +- src/dbms/database.hpp | 2 +- src/dbms/database_handler.hpp | 3 +- src/dbms/dbms_handler.cpp | 75 ++++ src/dbms/dbms_handler.hpp | 69 ++-- src/dbms/inmemory/storage_helper.hpp | 10 +- src/dbms/replication_client.cpp | 34 ++ src/dbms/replication_client.hpp | 21 ++ src/dbms/replication_handler.cpp | 164 ++++---- src/dbms/replication_handler.hpp | 6 +- src/memgraph.cpp | 25 +- src/query/frontend/ast/ast.hpp | 2 +- src/query/interpreter.cpp | 55 ++- src/replication/CMakeLists.txt | 4 + .../include/replication/messages.hpp | 44 +++ .../replication/replication_client.hpp | 82 ++++ .../replication/replication_server.hpp | 24 -- src/replication/include/replication/state.hpp | 28 +- src/replication/messages.cpp | 65 ++++ src/replication/replication_client.cpp | 40 ++ src/replication/replication_server.cpp | 46 +-- src/replication/state.cpp | 39 +- src/rpc/client.hpp | 57 ++- src/slk/streams.cpp | 8 +- src/slk/streams.hpp | 8 +- src/storage/v2/CMakeLists.txt | 3 +- src/storage/v2/disk/storage.hpp | 6 - src/storage/v2/durability/durability.cpp | 10 +- .../v2/inmemory/replication/recovery.cpp | 238 ++++++++++++ .../v2/inmemory/replication/recovery.hpp | 32 ++ .../replication/replication_client.cpp | 349 ------------------ .../replication/replication_client.hpp | 38 -- src/storage/v2/inmemory/storage.cpp | 16 +- src/storage/v2/inmemory/storage.hpp | 12 +- src/storage/v2/replication/enums.hpp | 2 +- src/storage/v2/replication/recovery.hpp | 28 ++ .../v2/replication/replication_client.cpp | 349 +++++++++--------- .../v2/replication/replication_client.hpp | 124 ++++--- .../replication/replication_storage_state.cpp | 18 +- .../replication/replication_storage_state.hpp | 25 +- src/storage/v2/storage.hpp | 10 +- .../show_while_creating_invalid_state.py | 155 +++++++- tests/integration/telemetry/client.cpp | 2 +- tests/unit/dbms_handler.cpp | 5 +- tests/unit/dbms_handler_community.cpp | 5 +- tests/unit/storage_v2_replication.cpp | 27 +- 49 files changed, 1398 insertions(+), 991 deletions(-) create mode 100644 src/dbms/dbms_handler.cpp create mode 100644 src/dbms/replication_client.cpp create mode 100644 src/dbms/replication_client.hpp create mode 100644 src/replication/include/replication/messages.hpp create mode 100644 src/replication/include/replication/replication_client.hpp create mode 100644 src/replication/messages.cpp create mode 100644 src/replication/replication_client.cpp create mode 100644 src/storage/v2/inmemory/replication/recovery.cpp create mode 100644 src/storage/v2/inmemory/replication/recovery.hpp delete mode 100644 src/storage/v2/inmemory/replication/replication_client.cpp delete mode 100644 src/storage/v2/inmemory/replication/replication_client.hpp create mode 100644 src/storage/v2/replication/recovery.hpp diff --git a/release/get_version.py b/release/get_version.py index cfce88475..a8539fab4 100755 --- a/release/get_version.py +++ b/release/get_version.py @@ -104,7 +104,9 @@ def retry(retry_limit, timeout=100): except Exception: time.sleep(timeout) return func(*args, **kwargs) + return wrapper + return inner_func @@ -200,19 +202,19 @@ if args.version: try: current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD") if current_branch != "master": - branches = get_output("git", "branch") - if "master" in branches: + branches = get_output("git", "branch", "-r", "--list", "origin/master") + if "origin/master" in branches: # If master is present locally, the fetch is allowed to fail # because this script will still be able to compare against the # master branch. try: - get_output("git", "fetch", "origin", "master:master") + get_output("git", "fetch", "origin", "master") except Exception: pass else: # If master is not present locally, the fetch command has to # succeed because something else will fail otherwise. - get_output("git", "fetch", "origin", "master:master") + get_output("git", "fetch", "origin", "master") except Exception: print("Fatal error while ensuring local master branch.") sys.exit(1) @@ -232,7 +234,7 @@ for branch in branches: match = branch_regex.match(branch) if match is not None: version = tuple(map(int, match.group(1).split("."))) - master_branch_merge = get_output("git", "merge-base", "master", branch) + master_branch_merge = get_output("git", "merge-base", "origin/master", branch) versions.append((version, branch, master_branch_merge)) versions.sort(reverse=True) @@ -243,7 +245,7 @@ current_version = None for version in versions: version_tuple, branch, master_branch_merge = version current_branch_merge = get_output("git", "merge-base", current_hash, branch) - master_current_merge = get_output("git", "merge-base", current_hash, "master") + master_current_merge = get_output("git", "merge-base", current_hash, "origin/master") # The first check checks whether this commit is a child of `master` and # the version branch was created before us. # The second check checks whether this commit is a child of the version diff --git a/src/dbms/CMakeLists.txt b/src/dbms/CMakeLists.txt index 8ec1e0972..f1df4985a 100644 --- a/src/dbms/CMakeLists.txt +++ b/src/dbms/CMakeLists.txt @@ -1,3 +1,3 @@ -add_library(mg-dbms STATIC database.cpp replication_handler.cpp inmemory/replication_handlers.cpp) +add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp) target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query) diff --git a/src/dbms/constants.hpp b/src/dbms/constants.hpp index 3ca61056b..e7ea9987b 100644 --- a/src/dbms/constants.hpp +++ b/src/dbms/constants.hpp @@ -15,4 +15,10 @@ namespace memgraph::dbms { constexpr static const char *kDefaultDB = "memgraph"; //!< Name of the default database +#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY +constexpr bool allow_mt_repl = true; +#else +constexpr bool allow_mt_repl = false; +#endif + } // namespace memgraph::dbms diff --git a/src/dbms/database.cpp b/src/dbms/database.cpp index 411e282e8..74ee13892 100644 --- a/src/dbms/database.cpp +++ b/src/dbms/database.cpp @@ -21,7 +21,7 @@ template struct memgraph::utils::Gatekeeper; namespace memgraph::dbms { -Database::Database(storage::Config config, const replication::ReplicationState &repl_state) +Database::Database(storage::Config config, replication::ReplicationState &repl_state) : trigger_store_(config.durability.storage_directory / "triggers"), streams_{config.durability.storage_directory / "streams"}, plan_cache_{FLAGS_query_plan_cache_max_size}, diff --git a/src/dbms/database.hpp b/src/dbms/database.hpp index 457aa1c1d..416ff76bc 100644 --- a/src/dbms/database.hpp +++ b/src/dbms/database.hpp @@ -48,7 +48,7 @@ class Database { * * @param config storage configuration */ - explicit Database(storage::Config config, const replication::ReplicationState &repl_state); + explicit Database(storage::Config config, replication::ReplicationState &repl_state); /** * @brief Returns the raw storage pointer. diff --git a/src/dbms/database_handler.hpp b/src/dbms/database_handler.hpp index a6b3b563b..617e614c3 100644 --- a/src/dbms/database_handler.hpp +++ b/src/dbms/database_handler.hpp @@ -51,8 +51,7 @@ class DatabaseHandler : public Handler { * @param config Storage configuration * @return HandlerT::NewResult */ - HandlerT::NewResult New(std::string_view name, storage::Config config, - const replication::ReplicationState &repl_state) { + HandlerT::NewResult New(std::string_view name, storage::Config config, replication::ReplicationState &repl_state) { // Control that no one is using the same data directory if (std::any_of(begin(), end(), [&](auto &elem) { auto db_acc = elem.second.access(); diff --git a/src/dbms/dbms_handler.cpp b/src/dbms/dbms_handler.cpp new file mode 100644 index 000000000..0af9364bf --- /dev/null +++ b/src/dbms/dbms_handler.cpp @@ -0,0 +1,75 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "dbms/dbms_handler.hpp" + +namespace memgraph::dbms { +#ifdef MG_ENTERPRISE +DbmsHandler::DbmsHandler( + storage::Config config, + memgraph::utils::Synchronized *auth, + bool recovery_on_startup, bool delete_on_drop) + : default_config_{std::move(config)}, + delete_on_drop_(delete_on_drop), + repl_state_{ReplicationStateRootPath(default_config_)} { + // TODO: Decouple storage config from dbms config + // TODO: Save individual db configs inside the kvstore and restore from there + storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases"); + const auto &db_dir = default_config_.durability.storage_directory; + const auto durability_dir = db_dir / ".durability"; + utils::EnsureDirOrDie(db_dir); + utils::EnsureDirOrDie(durability_dir); + durability_ = std::make_unique(durability_dir); + + // Generate the default database + MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB."); + + // Recover previous databases + if (recovery_on_startup) { + for (const auto &[name, _] : *durability_) { + if (name == kDefaultDB) continue; // Already set + spdlog::info("Restoring database {}.", name); + MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name); + spdlog::info("Database {} restored.", name); + } + } else { // Clear databases from the durability list and auth + auto locked_auth = auth->Lock(); + for (const auto &[name, _] : *durability_) { + if (name == kDefaultDB) continue; + locked_auth->DeleteDatabase(name); + durability_->Delete(name); + } + } + + // Startup replication state (if recovered at startup) + auto replica = [this](replication::RoleReplicaData const &data) { + // Register handlers + InMemoryReplicationHandlers::Register(this, *data.server); + if (!data.server->Start()) { + spdlog::error("Unable to start the replication server."); + return false; + } + return true; + }; + // Replication frequent check start + auto main = [this](replication::RoleMainData &data) { + for (auto &client : data.registered_replicas_) { + StartReplicaClient(*this, client); + } + return true; + }; + // Startup proccess for main/replica + MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()), + "Replica recovery failure!"); +} +#endif + +} // namespace memgraph::dbms diff --git a/src/dbms/dbms_handler.hpp b/src/dbms/dbms_handler.hpp index 27ab963d4..3151398ab 100644 --- a/src/dbms/dbms_handler.hpp +++ b/src/dbms/dbms_handler.hpp @@ -26,9 +26,11 @@ #include "auth/auth.hpp" #include "constants.hpp" #include "dbms/database.hpp" +#include "dbms/inmemory/replication_handlers.hpp" #ifdef MG_ENTERPRISE #include "dbms/database_handler.hpp" #endif +#include "dbms/replication_client.hpp" #include "global.hpp" #include "query/config.hpp" #include "query/interpreter_context.hpp" @@ -102,52 +104,22 @@ class DbmsHandler { * @param recovery_on_startup restore databases (and its content) and authentication data * @param delete_on_drop when dropping delete any associated directories on disk */ - DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state, auto *auth, - bool recovery_on_startup, bool delete_on_drop) - : lock_{utils::RWLock::Priority::READ}, - default_config_{std::move(config)}, - repl_state_(repl_state), - delete_on_drop_(delete_on_drop) { - // TODO: Decouple storage config from dbms config - // TODO: Save individual db configs inside the kvstore and restore from there - storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases"); - const auto &db_dir = default_config_.durability.storage_directory; - const auto durability_dir = db_dir / ".durability"; - utils::EnsureDirOrDie(db_dir); - utils::EnsureDirOrDie(durability_dir); - durability_ = std::make_unique(durability_dir); - - // Generate the default database - MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB."); - // Recover previous databases - if (recovery_on_startup) { - for (const auto &[name, _] : *durability_) { - if (name == kDefaultDB) continue; // Already set - spdlog::info("Restoring database {}.", name); - MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name); - spdlog::info("Database {} restored.", name); - } - } else { // Clear databases from the durability list and auth - auto locked_auth = auth->Lock(); - for (const auto &[name, _] : *durability_) { - if (name == kDefaultDB) continue; - locked_auth->DeleteDatabase(name); - durability_->Delete(name); - } - } - } + DbmsHandler(storage::Config config, + memgraph::utils::Synchronized *auth, + bool recovery_on_startup, bool delete_on_drop); // TODO If more arguments are added use a config strut #else /** * @brief Initialize the handler. A single database is supported in community edition. * * @param configs storage configuration */ - DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state) - : db_gatekeeper_{[&] { + DbmsHandler(storage::Config config) + : repl_state_{ReplicationStateRootPath(config)}, + db_gatekeeper_{[&] { config.name = kDefaultDB; return std::move(config); }(), - repl_state} {} + repl_state_} {} #endif #ifdef MG_ENTERPRISE @@ -248,6 +220,12 @@ class DbmsHandler { #endif } + replication::ReplicationState &ReplicationState() { return repl_state_; } + replication::ReplicationState const &ReplicationState() const { return repl_state_; } + + bool IsMain() const { return repl_state_.IsMain(); } + bool IsReplica() const { return repl_state_.IsReplica(); } + /** * @brief Return the statistics all databases. * @@ -536,14 +514,15 @@ class DbmsHandler { throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name); } - mutable LockT lock_; //!< protective lock - storage::Config default_config_; //!< Storage configuration used when creating new databases - const replication::ReplicationState &repl_state_; //!< Global replication state - DatabaseHandler db_handler_; //!< multi-tenancy storage handler - std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) - bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory - std::set> defunct_dbs_; //!< Databases that are in an unknown state due to various failures -#else + mutable LockT lock_{utils::RWLock::Priority::READ}; //!< protective lock + storage::Config default_config_; //!< Storage configuration used when creating new databases + DatabaseHandler db_handler_; //!< multi-tenancy storage handler + std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) + bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory + std::set defunct_dbs_; //!< Databases that are in an unknown state due to various failures +#endif + replication::ReplicationState repl_state_; //!< Global replication state +#ifndef MG_ENTERPRISE mutable utils::Gatekeeper db_gatekeeper_; //!< Single databases gatekeeper #endif }; diff --git a/src/dbms/inmemory/storage_helper.hpp b/src/dbms/inmemory/storage_helper.hpp index 347c16928..1cd9f9f4e 100644 --- a/src/dbms/inmemory/storage_helper.hpp +++ b/src/dbms/inmemory/storage_helper.hpp @@ -22,14 +22,8 @@ namespace memgraph::dbms { -#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY -constexpr bool allow_mt_repl = true; -#else -constexpr bool allow_mt_repl = false; -#endif - -inline std::unique_ptr CreateInMemoryStorage( - storage::Config config, const ::memgraph::replication::ReplicationState &repl_state) { +inline std::unique_ptr CreateInMemoryStorage(storage::Config config, + ::memgraph::replication::ReplicationState &repl_state) { const auto wal_mode = config.durability.snapshot_wal_mode; const auto name = config.name; auto storage = std::make_unique(std::move(config)); diff --git a/src/dbms/replication_client.cpp b/src/dbms/replication_client.cpp new file mode 100644 index 000000000..bfa4c622f --- /dev/null +++ b/src/dbms/replication_client.cpp @@ -0,0 +1,34 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "dbms/replication_client.hpp" + +namespace memgraph::dbms { + +void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client) { + // No client error, start instance level client + auto const &endpoint = client.rpc_client_.Endpoint(); + spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); + client.StartFrequentCheck([&dbms_handler](std::string_view name) { + // Working connection, check if any database has been left behind + dbms_handler.ForEach([name](dbms::Database *db) { + // Specific database <-> replica client + db->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) { + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { + // Database <-> replica might be behind, check and recover + client->TryCheckReplicaStateAsync(db->storage()); + } + }); + }); + }); +} + +} // namespace memgraph::dbms diff --git a/src/dbms/replication_client.hpp b/src/dbms/replication_client.hpp new file mode 100644 index 000000000..c1bac91a2 --- /dev/null +++ b/src/dbms/replication_client.hpp @@ -0,0 +1,21 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "dbms/dbms_handler.hpp" +#include "replication/replication_client.hpp" + +namespace memgraph::dbms { + +void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client); + +} // namespace memgraph::dbms diff --git a/src/dbms/replication_handler.cpp b/src/dbms/replication_handler.cpp index cff93fd6b..2cbe2c432 100644 --- a/src/dbms/replication_handler.cpp +++ b/src/dbms/replication_handler.cpp @@ -15,6 +15,7 @@ #include "dbms/dbms_handler.hpp" #include "dbms/inmemory/replication_handlers.hpp" #include "dbms/inmemory/storage_helper.hpp" +#include "dbms/replication_client.hpp" #include "replication/state.hpp" using memgraph::replication::ReplicationClientConfig; @@ -41,6 +42,8 @@ std::string RegisterReplicaErrorToString(RegisterReplicaError error) { } } // namespace +ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {} + bool ReplicationHandler::SetReplicationRoleMain() { auto const main_handler = [](RoleMainData const &) { // If we are already MAIN, we don't want to change anything @@ -56,42 +59,49 @@ bool ReplicationHandler::SetReplicationRoleMain() { // STEP 2) Change to MAIN // TODO: restore replication servers if false? - if (!repl_state_.SetReplicationRoleMain()) { + if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) { // TODO: Handle recovery on failure??? return false; } // STEP 3) We are now MAIN, update storage local epoch + const auto &epoch = + std::get(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_; dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); - storage->repl_storage_state_.epoch_ = std::get(std::as_const(repl_state_).ReplicationData()).epoch_; + storage->repl_storage_state_.epoch_ = epoch; }); return true; }; // TODO: under lock - return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData()); + return std::visit(utils::Overloaded{main_handler, replica_handler}, + dbms_handler_.ReplicationState().ReplicationData()); } bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) { // We don't want to restart the server if we're already a REPLICA - if (repl_state_.IsReplica()) { + if (dbms_handler_.ReplicationState().IsReplica()) { return false; } - // Remove registered replicas + // TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are + // deleting the replica. + // Remove database specific clients dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); }); }); + // Remove instance level clients + std::get(dbms_handler_.ReplicationState().ReplicationData()).registered_replicas_.clear(); // Creates the server - repl_state_.SetReplicationRoleReplica(config); + dbms_handler_.ReplicationState().SetReplicationRoleReplica(config); // Start const auto success = - std::visit(utils::Overloaded{[](auto) { + std::visit(utils::Overloaded{[](RoleMainData const &) { // ASSERT return false; }, @@ -104,36 +114,37 @@ bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication:: } return true; }}, - repl_state_.ReplicationData()); + dbms_handler_.ReplicationState().ReplicationData()); // TODO Handle error (restore to main?) return success; } auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config) -> memgraph::utils::BasicResult { - MG_ASSERT(repl_state_.IsMain(), "Only main instance can register a replica!"); + MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!"); - auto res = repl_state_.RegisterReplica(config); - switch (res) { - case memgraph::replication::RegisterReplicaError::NOT_MAIN: - MG_ASSERT(false, "Only main instance can register a replica!"); - return {}; - case memgraph::replication::RegisterReplicaError::NAME_EXISTS: - return memgraph::dbms::RegisterReplicaError::NAME_EXISTS; - case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS: - return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS; - case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED: - return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED; - case memgraph::replication::RegisterReplicaError::SUCCESS: - break; - } - - bool all_clients_good = true; + auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config); + if (instance_client.HasError()) switch (instance_client.GetError()) { + case memgraph::replication::RegisterReplicaError::NOT_MAIN: + MG_ASSERT(false, "Only main instance can register a replica!"); + return {}; + case memgraph::replication::RegisterReplicaError::NAME_EXISTS: + return memgraph::dbms::RegisterReplicaError::NAME_EXISTS; + case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS: + return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS; + case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED: + return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED; + case memgraph::replication::RegisterReplicaError::SUCCESS: + break; + } if (!allow_mt_repl && dbms_handler_.All().size() > 1) { spdlog::warn("Multi-tenant replication is currently not supported!"); } + bool all_clients_good = true; + + // Add database specific clients (NOTE Currently all databases are connected to each replica) dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); if (!allow_mt_repl && storage->id() != kDefaultDB) { @@ -143,18 +154,29 @@ auto ReplicationHandler::RegisterReplica(const memgraph::replication::Replicatio if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return; all_clients_good &= - storage->repl_storage_state_.replication_clients_.WithLock([storage, &config](auto &clients) -> bool { - auto client = storage->CreateReplicationClient(config, &storage->repl_storage_state_.epoch_); - client->Start(); - - if (client->State() == storage::replication::ReplicaState::INVALID) { + storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) { + auto client = std::make_unique(*instance_client.GetValue()); + client->Start(storage); + // After start the storage <-> replica state should be READY or RECOVERING (if correctly started) + // MAYBE_BEHIND isn't a statement of the current state, this is the default value + // Failed to start due to branching of MAIN and REPLICA + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { return false; } - clients.push_back(std::move(client)); + storage_clients.push_back(std::move(client)); return true; }); }); - if (!all_clients_good) return RegisterReplicaError::CONNECTION_FAILED; // TODO: this happen to 1 or many...what to do + + // NOTE Currently if any databases fails, we revert back + if (!all_clients_good) { + spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name); + UnregisterReplica(config.name); + return RegisterReplicaError::CONNECTION_FAILED; + } + + // No client error, start instance level client + StartReplicaClient(dbms_handler_, *instance_client.GetValue()); return {}; } @@ -163,60 +185,66 @@ auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterR return UnregisterReplicaResult::NOT_MAIN; }; auto const main_handler = [this, name](RoleMainData &mainData) -> UnregisterReplicaResult { - if (!repl_state_.TryPersistUnregisterReplica(name)) { + if (!dbms_handler_.ReplicationState().TryPersistUnregisterReplica(name)) { return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED; } - auto const n_unregistered = - std::erase_if(mainData.registered_replicas_, - [&](ReplicationClientConfig const ®istered_config) { return registered_config.name == name; }); - - dbms_handler_.ForEach([&](Database *db) { - db->storage()->repl_storage_state_.replication_clients_.WithLock( - [&](auto &clients) { std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); }); + // Remove database specific clients + dbms_handler_.ForEach([name](Database *db) { + db->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) { + std::erase_if(clients, [name](const auto &client) { return client->Name() == name; }); + }); }); - + // Remove instance level clients + auto const n_unregistered = + std::erase_if(mainData.registered_replicas_, [name](auto const &client) { return client.name_ == name; }); return n_unregistered != 0 ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER; }; - return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData()); + return std::visit(utils::Overloaded{main_handler, replica_handler}, + dbms_handler_.ReplicationState().ReplicationData()); } -auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { return repl_state_.GetRole(); } +auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { + return dbms_handler_.ReplicationState().GetRole(); +} -bool ReplicationHandler::IsMain() const { return repl_state_.IsMain(); } +bool ReplicationHandler::IsMain() const { return dbms_handler_.ReplicationState().IsMain(); } -bool ReplicationHandler::IsReplica() const { return repl_state_.IsReplica(); } +bool ReplicationHandler::IsReplica() const { return dbms_handler_.ReplicationState().IsReplica(); } -void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage) { +// Per storage +// NOTE Storage will connect to all replicas. Future work might change this +void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage) { spdlog::info("Restoring replication role."); /// MAIN - auto const recover_main = [&storage](RoleMainData const &mainData) { - for (const auto &config : mainData.registered_replicas_) { - spdlog::info("Replica {} restoration started for {}.", config.name, storage.id()); + auto const recover_main = [&storage](RoleMainData &mainData) { + // Each individual client has already been restored and started. Here we just go through each database and start its + // client + for (auto &instance_client : mainData.registered_replicas_) { + spdlog::info("Replica {} restoration started for {}.", instance_client.name_, storage.id()); - auto register_replica = [&storage](const memgraph::replication::ReplicationClientConfig &config) - -> memgraph::utils::BasicResult { - return storage.repl_storage_state_.replication_clients_.WithLock( - [&storage, &config](auto &clients) -> utils::BasicResult { - auto client = storage.CreateReplicationClient(config, &storage.repl_storage_state_.epoch_); - client->Start(); + const auto &ret = storage.repl_storage_state_.replication_clients_.WithLock( + [&](auto &storage_clients) -> utils::BasicResult { + auto client = std::make_unique(instance_client); + client->Start(&storage); + // After start the storage <-> replica state should be READY or RECOVERING (if correctly started) + // MAYBE_BEHIND isn't a statement of the current state, this is the default value + // Failed to start due to branching of MAIN and REPLICA + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { + spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", + instance_client.name_); + } + storage_clients.push_back(std::move(client)); + return {}; + }); - if (client->State() == storage::replication::ReplicaState::INVALID) { - spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", - client->Name()); - } - clients.push_back(std::move(client)); - return {}; - }); - }; - - auto ret = register_replica(config); if (ret.HasError()) { MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError()); - LOG_FATAL("Failure when restoring replica {}: {}.", config.name, RegisterReplicaErrorToString(ret.GetError())); + LOG_FATAL("Failure when restoring replica {}: {}.", instance_client.name_, + RegisterReplicaErrorToString(ret.GetError())); } - spdlog::info("Replica {} restored for {}.", config.name, storage.id()); + spdlog::info("Replica {} restored for {}.", instance_client.name_, storage.id()); } spdlog::info("Replication role restored to MAIN."); }; @@ -229,6 +257,6 @@ void RestoreReplication(const replication::ReplicationState &repl_state, storage recover_main, recover_replica, }, - std::as_const(repl_state).ReplicationData()); + repl_state.ReplicationData()); } } // namespace memgraph::dbms diff --git a/src/dbms/replication_handler.hpp b/src/dbms/replication_handler.hpp index e50c47969..dc95407b1 100644 --- a/src/dbms/replication_handler.hpp +++ b/src/dbms/replication_handler.hpp @@ -36,8 +36,7 @@ enum class UnregisterReplicaResult : uint8_t { /// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage /// TODO: extend to do multiple storages struct ReplicationHandler { - ReplicationHandler(memgraph::replication::ReplicationState &replState, DbmsHandler &dbms_handler) - : repl_state_(replState), dbms_handler_(dbms_handler) {} + explicit ReplicationHandler(DbmsHandler &dbms_handler); // as REPLICA, become MAIN bool SetReplicationRoleMain(); @@ -58,12 +57,11 @@ struct ReplicationHandler { bool IsReplica() const; private: - memgraph::replication::ReplicationState &repl_state_; DbmsHandler &dbms_handler_; }; /// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage /// TODO: extend to do multiple storages -void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage); +void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage); } // namespace memgraph::dbms diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 983dd61f9..ce43f634d 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -368,34 +368,17 @@ int main(int argc, char **argv) { std::unique_ptr auth_checker; auth_glue(&auth_, auth_handler, auth_checker); - memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(db_config)); - - memgraph::dbms::DbmsHandler dbms_handler(db_config, repl_state + memgraph::dbms::DbmsHandler dbms_handler(db_config #ifdef MG_ENTERPRISE , &auth_, FLAGS_data_recovery_on_startup, FLAGS_storage_delete_on_drop #endif ); auto db_acc = dbms_handler.Get(); - memgraph::query::InterpreterContext interpreter_context_(interp_config, &dbms_handler, &repl_state, - auth_handler.get(), auth_checker.get()); - MG_ASSERT(db_acc, "Failed to access the main database"); - // TODO: Move it somewhere better - // Startup replication state (if recovered at startup) - MG_ASSERT(std::visit(memgraph::utils::Overloaded{[](memgraph::replication::RoleMainData const &) { return true; }, - [&](memgraph::replication::RoleReplicaData const &data) { - // Register handlers - memgraph::dbms::InMemoryReplicationHandlers::Register( - &dbms_handler, *data.server); - if (!data.server->Start()) { - spdlog::error("Unable to start the replication server."); - return false; - } - return true; - }}, - repl_state.ReplicationData()), - "Replica recovery failure!"); + memgraph::query::InterpreterContext interpreter_context_( + interp_config, &dbms_handler, &dbms_handler.ReplicationState(), auth_handler.get(), auth_checker.get()); + MG_ASSERT(db_acc, "Failed to access the main database"); memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(), FLAGS_data_directory); diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index dc11c3887..d63736c85 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -3025,7 +3025,7 @@ class ReplicationQuery : public memgraph::query::Query { enum class SyncMode { SYNC, ASYNC }; - enum class ReplicaState { READY, REPLICATING, RECOVERY, INVALID }; + enum class ReplicaState { READY, REPLICATING, RECOVERY, MAYBE_BEHIND }; ReplicationQuery() = default; diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 354f13dc3..d957d4c2e 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -274,8 +274,7 @@ inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode class ReplQueryHandler final : public query::ReplicationQueryHandler { public: - explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler, memgraph::replication::ReplicationState *repl_state) - : dbms_handler_(dbms_handler), handler_{*repl_state, *dbms_handler} {} + explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler) : dbms_handler_(dbms_handler), handler_{*dbms_handler} {} /// @throw QueryRuntimeException if an error ocurred. void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { @@ -404,8 +403,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { case storage::replication::ReplicaState::RECOVERY: replica.state = ReplicationQuery::ReplicaState::RECOVERY; break; - case storage::replication::ReplicaState::INVALID: - replica.state = ReplicationQuery::ReplicaState::INVALID; + case storage::replication::ReplicaState::MAYBE_BEHIND: + replica.state = ReplicationQuery::ReplicaState::MAYBE_BEHIND; break; } @@ -713,8 +712,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters, dbms::DbmsHandler *dbms_handler, const query::InterpreterConfig &config, - std::vector *notifications, - memgraph::replication::ReplicationState *repl_state) { + std::vector *notifications) { // TODO: MemoryResource for EvaluationContext, it should probably be passed as // the argument to Callback. EvaluationContext evaluation_context; @@ -734,8 +732,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & notifications->emplace_back(SeverityLevel::WARNING, NotificationCode::REPLICA_PORT_WARNING, "Be careful the replication port must be different from the memgraph port!"); } - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, role = repl_query->role_, - maybe_port]() mutable { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, role = repl_query->role_, maybe_port]() mutable { handler.SetReplicationRole(role, maybe_port); return std::vector>(); }; @@ -747,7 +744,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: { callback.header = {"replication role"}; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}] { + callback.fn = [handler = ReplQueryHandler{dbms_handler}] { auto mode = handler.ShowReplicationRole(); switch (mode) { case ReplicationQuery::ReplicationRole::MAIN: { @@ -766,7 +763,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & auto socket_address = repl_query->socket_address_->Accept(evaluator); const auto replica_check_frequency = config.replication_replica_check_frequency; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, name, socket_address, sync_mode, + callback.fn = [handler = ReplQueryHandler{dbms_handler}, name, socket_address, sync_mode, replica_check_frequency]() mutable { handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency); return std::vector>(); @@ -777,7 +774,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } case ReplicationQuery::Action::DROP_REPLICA: { const auto &name = repl_query->replica_name_; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, name]() mutable { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, name]() mutable { handler.DropReplica(name); return std::vector>(); }; @@ -789,7 +786,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & callback.header = { "name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master", "state"}; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, replica_nfields = callback.header.size()] { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, replica_nfields = callback.header.size()] { const auto &replicas = handler.ShowReplicas(); auto typed_replicas = std::vector>{}; typed_replicas.reserve(replicas.size()); @@ -822,7 +819,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & case ReplicationQuery::ReplicaState::RECOVERY: typed_replica.emplace_back("recovery"); break; - case ReplicationQuery::ReplicaState::INVALID: + case ReplicationQuery::ReplicaState::MAYBE_BEHIND: typed_replica.emplace_back("invalid"); break; } @@ -2263,15 +2260,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit_transaction, std::vector *notifications, dbms::DbmsHandler &dbms_handler, - const InterpreterConfig &config, - memgraph::replication::ReplicationState *repl_state) { + const InterpreterConfig &config) { if (in_explicit_transaction) { throw ReplicationModificationInMulticommandTxException(); } auto *replication_query = utils::Downcast(parsed_query.query); - auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, - notifications, repl_state); + auto callback = + HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, notifications); return PreparedQuery{callback.header, std::move(parsed_query.required_privileges), [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( @@ -3349,8 +3345,7 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB ¤t_db, InterpreterContext *interpreter_context, - std::optional> on_change_cb, - memgraph::replication::ReplicationState *repl_state) { + std::optional> on_change_cb) { #ifdef MG_ENTERPRISE if (!license::global_license_checker.IsEnterpriseValidFast()) { throw QueryException("Trying to use enterprise feature without a valid license."); @@ -3361,9 +3356,11 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur auto *query = utils::Downcast(parsed_query.query); auto *db_handler = interpreter_context->dbms_handler; + const bool is_replica = interpreter_context->repl_state->IsReplica(); + switch (query->action_) { case MultiDatabaseQuery::Action::CREATE: - if (repl_state->IsReplica()) { + if (is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{ @@ -3408,12 +3405,12 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur if (current_db.in_explicit_db_) { throw QueryException("Database switching is prohibited if session explicitly defines the used database"); } - if (!dbms::allow_mt_repl && repl_state->IsReplica()) { + if (!dbms::allow_mt_repl && is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{{"STATUS"}, std::move(parsed_query.required_privileges), - [db_name = query->db_name_, db_handler, ¤t_db, on_change_cb]( + [db_name = query->db_name_, db_handler, ¤t_db, on_change = std::move(on_change_cb)]( AnyStream *stream, std::optional n) -> std::optional { std::vector> status; std::string res; @@ -3423,7 +3420,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur res = "Already using " + db_name; } else { auto tmp = db_handler->Get(db_name); - if (on_change_cb) (*on_change_cb)(db_name); // Will trow if cb fails + if (on_change) (*on_change)(db_name); // Will trow if cb fails current_db.SetCurrentDB(std::move(tmp), false); res = "Using " + db_name; } @@ -3442,7 +3439,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur query->db_name_}; case MultiDatabaseQuery::Action::DROP: - if (repl_state->IsReplica()) { + if (is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{ @@ -3765,9 +3762,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, &query_execution->notifications, current_db_); } else if (utils::Downcast(parsed_query.query)) { /// TODO: make replication DB agnostic - prepared_query = PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, - &query_execution->notifications, *interpreter_context_->dbms_handler, - interpreter_context_->config, interpreter_context_->repl_state); + prepared_query = + PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + *interpreter_context_->dbms_handler, interpreter_context_->config); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { @@ -3807,8 +3804,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, throw MultiDatabaseQueryInMulticommandTxException(); } /// SYSTEM (Replication) + INTERPRETER - prepared_query = PrepareMultiDatabaseQuery(std::move(parsed_query), current_db_, interpreter_context_, on_change_, - interpreter_context_->repl_state); + prepared_query = + PrepareMultiDatabaseQuery(std::move(parsed_query), current_db_, interpreter_context_, on_change_); } else if (utils::Downcast(parsed_query.query)) { /// SYSTEM PURE ("SHOW DATABASES") /// INTERPRETER (TODO: "SHOW DATABASE") diff --git a/src/replication/CMakeLists.txt b/src/replication/CMakeLists.txt index 772ae5591..597ed096a 100644 --- a/src/replication/CMakeLists.txt +++ b/src/replication/CMakeLists.txt @@ -6,8 +6,10 @@ target_sources(mg-replication include/replication/epoch.hpp include/replication/config.hpp include/replication/mode.hpp + include/replication/messages.hpp include/replication/role.hpp include/replication/status.hpp + include/replication/replication_client.hpp include/replication/replication_server.hpp PRIVATE @@ -15,6 +17,8 @@ target_sources(mg-replication epoch.cpp config.cpp status.cpp + messages.cpp + replication_client.cpp replication_server.cpp ) target_include_directories(mg-replication PUBLIC include) diff --git a/src/replication/include/replication/messages.hpp b/src/replication/include/replication/messages.hpp new file mode 100644 index 000000000..57cf29351 --- /dev/null +++ b/src/replication/include/replication/messages.hpp @@ -0,0 +1,44 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "rpc/messages.hpp" +#include "slk/serialization.hpp" + +namespace memgraph::replication { + +struct FrequentHeartbeatReq { + static const utils::TypeInfo kType; // TODO: make constexpr? + static const utils::TypeInfo &GetTypeInfo() { return kType; } // WHAT? + + static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); + static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); + FrequentHeartbeatReq() = default; +}; + +struct FrequentHeartbeatRes { + static const utils::TypeInfo kType; + static const utils::TypeInfo &GetTypeInfo() { return kType; } + + static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); + static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); + FrequentHeartbeatRes() = default; + explicit FrequentHeartbeatRes(bool success) : success(success) {} + + bool success; +}; + +using FrequentHeartbeatRpc = rpc::RequestResponse; + +void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder); + +} // namespace memgraph::replication diff --git a/src/replication/include/replication/replication_client.hpp b/src/replication/include/replication/replication_client.hpp new file mode 100644 index 000000000..16e1010bf --- /dev/null +++ b/src/replication/include/replication/replication_client.hpp @@ -0,0 +1,82 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "replication/config.hpp" +#include "replication/messages.hpp" +#include "rpc/client.hpp" +#include "utils/scheduler.hpp" +#include "utils/thread_pool.hpp" + +#include +#include + +namespace memgraph::replication { + +template +concept InvocableWithStringView = std::invocable; + +struct ReplicationClient { + explicit ReplicationClient(const memgraph::replication::ReplicationClientConfig &config); + + ~ReplicationClient(); + ReplicationClient(ReplicationClient const &) = delete; + ReplicationClient &operator=(ReplicationClient const &) = delete; + ReplicationClient(ReplicationClient &&) noexcept = delete; + ReplicationClient &operator=(ReplicationClient &&) noexcept = delete; + + template + void StartFrequentCheck(F &&callback) { + // Help the user to get the most accurate replica state possible. + if (replica_check_frequency_ > std::chrono::seconds(0)) { + replica_checker_.Run("Replica Checker", replica_check_frequency_, [this, cb = std::forward(callback)] { + try { + bool success = false; + { + auto stream{rpc_client_.Stream()}; + success = stream.AwaitResponse().success; + } + if (success) { + cb(name_); + } + } catch (const rpc::RpcFailedException &) { + // Nothing to do...wait for a reconnect + } + }); + } + } + + std::string name_; + communication::ClientContext rpc_context_; + rpc::Client rpc_client_; + std::chrono::seconds replica_check_frequency_; + + memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC}; + // This thread pool is used for background tasks so we don't + // block the main storage thread + // We use only 1 thread for 2 reasons: + // - background tasks ALWAYS contain some kind of RPC communication. + // We can't have multiple RPC communication from a same client + // because that's not logically valid (e.g. you cannot send a snapshot + // and WAL at a same time because WAL will arrive earlier and be applied + // before the snapshot which is not correct) + // - the implementation is simplified as we have a total control of what + // this pool is executing. Also, we can simply queue multiple tasks + // and be sure of the execution order. + // Not having mulitple possible threads in the same client allows us + // to ignore concurrency problems inside the client. + utils::ThreadPool thread_pool_{1}; + + utils::Scheduler replica_checker_; +}; + +} // namespace memgraph::replication diff --git a/src/replication/include/replication/replication_server.hpp b/src/replication/include/replication/replication_server.hpp index e9ca1b549..5ff41b8a5 100644 --- a/src/replication/include/replication/replication_server.hpp +++ b/src/replication/include/replication/replication_server.hpp @@ -17,30 +17,6 @@ namespace memgraph::replication { -struct FrequentHeartbeatReq { - static const utils::TypeInfo kType; // TODO: make constexpr? - static const utils::TypeInfo &GetTypeInfo() { return kType; } // WHAT? - - static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); - static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); - FrequentHeartbeatReq() = default; -}; - -struct FrequentHeartbeatRes { - static const utils::TypeInfo kType; - static const utils::TypeInfo &GetTypeInfo() { return kType; } - - static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); - static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); - FrequentHeartbeatRes() = default; - explicit FrequentHeartbeatRes(bool success) : success(success) {} - - bool success; -}; - -// TODO: move to own header -using FrequentHeartbeatRpc = rpc::RequestResponse; - class ReplicationServer { public: explicit ReplicationServer(const memgraph::replication::ReplicationServerConfig &config); diff --git a/src/replication/include/replication/state.hpp b/src/replication/include/replication/state.hpp index 0460d0a9d..76aec1053 100644 --- a/src/replication/include/replication/state.hpp +++ b/src/replication/include/replication/state.hpp @@ -11,19 +11,22 @@ #pragma once -#include -#include -#include -#include - #include "kvstore/kvstore.hpp" #include "replication/config.hpp" #include "replication/epoch.hpp" #include "replication/mode.hpp" +#include "replication/replication_client.hpp" #include "replication/role.hpp" #include "replication_server.hpp" #include "status.hpp" #include "utils/result.hpp" +#include "utils/synchronized.hpp" + +#include +#include +#include +#include +#include namespace memgraph::replication { @@ -32,8 +35,17 @@ enum class RolePersisted : uint8_t { UNKNOWN_OR_NO, YES }; enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, COULD_NOT_BE_PERSISTED, NOT_MAIN, SUCCESS }; struct RoleMainData { + RoleMainData() = default; + explicit RoleMainData(ReplicationEpoch e) : epoch_(std::move(e)) {} + ~RoleMainData() = default; + + RoleMainData(RoleMainData const &) = delete; + RoleMainData &operator=(RoleMainData const &) = delete; + RoleMainData(RoleMainData &&) = default; + RoleMainData &operator=(RoleMainData &&) = default; + ReplicationEpoch epoch_; - std::vector registered_replicas_; + std::list registered_replicas_{}; }; struct RoleReplicaData { @@ -41,8 +53,10 @@ struct RoleReplicaData { std::unique_ptr server; }; +// Global (instance) level object struct ReplicationState { explicit ReplicationState(std::optional durability_dir); + ~ReplicationState() = default; ReplicationState(ReplicationState const &) = delete; ReplicationState(ReplicationState &&) = delete; @@ -74,7 +88,7 @@ struct ReplicationState { // TODO: locked access auto ReplicationData() -> ReplicationData_t & { return replication_data_; } auto ReplicationData() const -> ReplicationData_t const & { return replication_data_; } - auto RegisterReplica(const ReplicationClientConfig &config) -> RegisterReplicaError; + utils::BasicResult RegisterReplica(const ReplicationClientConfig &config); bool SetReplicationRoleMain(); diff --git a/src/replication/messages.cpp b/src/replication/messages.cpp new file mode 100644 index 000000000..4503e9df2 --- /dev/null +++ b/src/replication/messages.cpp @@ -0,0 +1,65 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/messages.hpp" +#include "rpc/messages.hpp" +#include "slk/serialization.hpp" +#include "slk/streams.hpp" + +namespace memgraph::slk { +// Serialize code for FrequentHeartbeatRes +void Save(const memgraph::replication::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self.success, builder); +} +void Load(memgraph::replication::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(&self->success, reader); +} + +// Serialize code for FrequentHeartbeatReq +void Save(const memgraph::replication::FrequentHeartbeatReq & /*self*/, memgraph::slk::Builder * /*builder*/) { + /* Nothing to serialize */ +} +void Load(memgraph::replication::FrequentHeartbeatReq * /*self*/, memgraph::slk::Reader * /*reader*/) { + /* Nothing to serialize */ +} + +} // namespace memgraph::slk + +namespace memgraph::replication { + +constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_REQ, "FrequentHeartbeatReq", + nullptr}; + +constexpr utils::TypeInfo FrequentHeartbeatRes::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_RES, "FrequentHeartbeatRes", + nullptr}; + +void FrequentHeartbeatReq::Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self, builder); +} +void FrequentHeartbeatReq::Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(self, reader); +} +void FrequentHeartbeatRes::Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self, builder); +} +void FrequentHeartbeatRes::Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(self, reader); +} + +void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { + FrequentHeartbeatReq req; + FrequentHeartbeatReq::Load(&req, req_reader); + memgraph::slk::Load(&req, req_reader); + FrequentHeartbeatRes res{true}; + memgraph::slk::Save(res, res_builder); +} + +} // namespace memgraph::replication diff --git a/src/replication/replication_client.cpp b/src/replication/replication_client.cpp new file mode 100644 index 000000000..d14250c2a --- /dev/null +++ b/src/replication/replication_client.cpp @@ -0,0 +1,40 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/replication_client.hpp" + +namespace memgraph::replication { + +static auto CreateClientContext(const memgraph::replication::ReplicationClientConfig &config) + -> communication::ClientContext { + return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file} + : communication::ClientContext{}; +} + +ReplicationClient::ReplicationClient(const memgraph::replication::ReplicationClientConfig &config) + : name_{config.name}, + rpc_context_{CreateClientContext(config)}, + rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port), + &rpc_context_}, + replica_check_frequency_{config.replica_check_frequency}, + mode_{config.mode} {} + +ReplicationClient::~ReplicationClient() { + auto endpoint = rpc_client_.Endpoint(); + try { + spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port); + } catch (...) { + // Logging can throw. Not a big deal, just ignore. + } + thread_pool_.Shutdown(); +} + +} // namespace memgraph::replication diff --git a/src/replication/replication_server.cpp b/src/replication/replication_server.cpp index 7d0ff3cc2..f79ea2add 100644 --- a/src/replication/replication_server.cpp +++ b/src/replication/replication_server.cpp @@ -10,25 +10,7 @@ // licenses/APL.txt. #include "replication/replication_server.hpp" -#include "rpc/messages.hpp" -#include "slk/serialization.hpp" -#include "slk/streams.hpp" - -namespace memgraph::slk { - -// Serialize code for FrequentHeartbeatRes -void Save(const memgraph::replication::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self.success, builder); -} -void Load(memgraph::replication::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(&self->success, reader); -} - -// Serialize code for FrequentHeartbeatReq -void Save(const memgraph::replication::FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) {} -void Load(memgraph::replication::FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) {} - -} // namespace memgraph::slk +#include "replication/messages.hpp" namespace memgraph::replication { namespace { @@ -39,13 +21,6 @@ auto CreateServerContext(const memgraph::replication::ReplicationServerConfig &c : communication::ServerContext{}; } -void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - FrequentHeartbeatReq req; - memgraph::slk::Load(&req, req_reader); - FrequentHeartbeatRes res{true}; - memgraph::slk::Save(res, res_builder); -} - // NOTE: The replication server must have a single thread for processing // because there is no need for more processing threads - each replica can // have only a single main server. Also, the single-threaded guarantee @@ -53,25 +28,6 @@ void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder constexpr auto kReplicationServerThreads = 1; } // namespace -constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_REQ, "FrequentHeartbeatReq", - nullptr}; - -constexpr utils::TypeInfo FrequentHeartbeatRes::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_RES, "FrequentHeartbeatRes", - nullptr}; - -void FrequentHeartbeatReq::Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self, builder); -} -void FrequentHeartbeatReq::Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(self, reader); -} -void FrequentHeartbeatRes::Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self, builder); -} -void FrequentHeartbeatRes::Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(self, reader); -} - ReplicationServer::ReplicationServer(const memgraph::replication::ReplicationServerConfig &config) : rpc_server_context_{CreateServerContext(config)}, rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_, diff --git a/src/replication/state.cpp b/src/replication/state.cpp index 4551eba7e..60c390e17 100644 --- a/src/replication/state.cpp +++ b/src/replication/state.cpp @@ -11,9 +11,11 @@ #include "replication/state.hpp" +#include "replication/replication_client.hpp" #include "replication/replication_server.hpp" #include "replication/status.hpp" #include "utils/file.hpp" +#include "utils/result.hpp" #include "utils/variant_helpers.hpp" constexpr auto kReplicationDirectory = std::string_view{"replication"}; @@ -125,12 +127,9 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t { return std::visit( utils::Overloaded{ [&](durability::MainRole &&r) -> FetchReplicationResult_t { - auto res = RoleMainData{ - .epoch_ = std::move(r.epoch), - }; + auto res = RoleMainData{std::move(r.epoch)}; auto b = durability_->begin(durability::kReplicationReplicaPrefix); auto e = durability_->end(durability::kReplicationReplicaPrefix); - res.registered_replicas_.reserve(durability_->Size(durability::kReplicationReplicaPrefix)); for (; b != e; ++b) { auto const &[replica_name, replica_data] = *b; auto json = nlohmann::json::parse(replica_data, nullptr, false); @@ -141,7 +140,8 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t { if (key_name != data.config.name) { return FetchReplicationError::PARSE_ERROR; } - res.registered_replicas_.emplace_back(std::move(data.config)); + // Instance clients + res.registered_replicas_.emplace_back(data.config); } catch (...) { return FetchReplicationError::PARSE_ERROR; } @@ -221,7 +221,7 @@ bool ReplicationState::SetReplicationRoleMain() { if (!TryPersistRoleMain(new_epoch)) { return false; } - replication_data_ = RoleMainData{.epoch_ = ReplicationEpoch{new_epoch}}; + replication_data_ = RoleMainData{ReplicationEpoch{new_epoch}}; return true; } @@ -233,16 +233,14 @@ bool ReplicationState::SetReplicationRoleReplica(const ReplicationServerConfig & return true; } -auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> RegisterReplicaError { - auto const replica_handler = [](RoleReplicaData const &) -> RegisterReplicaError { - return RegisterReplicaError::NOT_MAIN; - }; - auto const main_handler = [this, &config](RoleMainData &mainData) -> RegisterReplicaError { +utils::BasicResult ReplicationState::RegisterReplica( + const ReplicationClientConfig &config) { + auto const replica_handler = [](RoleReplicaData const &) { return RegisterReplicaError::NOT_MAIN; }; + ReplicationClient *client{nullptr}; + auto const main_handler = [&client, &config, this](RoleMainData &mainData) -> RegisterReplicaError { // name check auto name_check = [&config](auto const &replicas) { - auto name_matches = [&name = config.name](ReplicationClientConfig const ®istered_config) { - return registered_config.name == name; - }; + auto name_matches = [&name = config.name](auto const &replica) { return replica.name_ == name; }; return std::any_of(replicas.begin(), replicas.end(), name_matches); }; if (name_check(mainData.registered_replicas_)) { @@ -251,8 +249,9 @@ auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> // endpoint check auto endpoint_check = [&](auto const &replicas) { - auto endpoint_matches = [&config](ReplicationClientConfig const ®istered_config) { - return registered_config.ip_address == config.ip_address && registered_config.port == config.port; + auto endpoint_matches = [&config](auto const &replica) { + const auto &ep = replica.rpc_client_.Endpoint(); + return ep.address == config.ip_address && ep.port == config.port; }; return std::any_of(replicas.begin(), replicas.end(), endpoint_matches); }; @@ -266,10 +265,14 @@ auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> } // set - mainData.registered_replicas_.emplace_back(config); + client = &mainData.registered_replicas_.emplace_back(config); return RegisterReplicaError::SUCCESS; }; - return std::visit(utils::Overloaded{main_handler, replica_handler}, replication_data_); + const auto &res = std::visit(utils::Overloaded{main_handler, replica_handler}, replication_data_); + if (res == RegisterReplicaError::SUCCESS) { + return client; + } + return res; } } // namespace memgraph::replication diff --git a/src/rpc/client.hpp b/src/rpc/client.hpp index f727391ac..1fd3fff8d 100644 --- a/src/rpc/client.hpp +++ b/src/rpc/client.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "communication/client.hpp" #include "io/network/endpoint.hpp" @@ -41,16 +42,25 @@ class Client { StreamHandler(Client *self, std::unique_lock &&guard, std::function res_load) - : self_(self), - guard_(std::move(guard)), - req_builder_([self](const uint8_t *data, size_t size, bool have_more) { - if (!self->client_->Write(data, size, have_more)) throw GenericRpcFailedException(); - }), - res_load_(res_load) {} + : self_(self), guard_(std::move(guard)), req_builder_(GenBuilderCallback(self, this)), res_load_(res_load) {} public: - StreamHandler(StreamHandler &&) noexcept = default; - StreamHandler &operator=(StreamHandler &&) noexcept = default; + StreamHandler(StreamHandler &&other) noexcept + : self_{std::exchange(other.self_, nullptr)}, + defunct_{std::exchange(other.defunct_, true)}, + guard_{std::move(other.guard_)}, + req_builder_{std::move(other.req_builder_), GenBuilderCallback(self_, this)}, + res_load_{std::move(other.res_load_)} {} + StreamHandler &operator=(StreamHandler &&other) noexcept { + if (&other != this) { + self_ = std::exchange(other.self_, nullptr); + defunct_ = std::exchange(other.defunct_, true); + guard_ = std::move(other.guard_); + req_builder_ = slk::Builder(std::move(other.req_builder_, GenBuilderCallback(self_, this))); + res_load_ = std::move(other.res_load_); + } + return *this; + } StreamHandler(const StreamHandler &) = delete; StreamHandler &operator=(const StreamHandler &) = delete; @@ -70,10 +80,18 @@ class Client { while (true) { auto ret = slk::CheckStreamComplete(self_->client_->GetData(), self_->client_->GetDataSize()); if (ret.status == slk::StreamStatus::INVALID) { + // Logically invalid state, connection is still up, defunct stream and release + defunct_ = true; + guard_.unlock(); throw GenericRpcFailedException(); - } else if (ret.status == slk::StreamStatus::PARTIAL) { + } + if (ret.status == slk::StreamStatus::PARTIAL) { if (!self_->client_->Read(ret.stream_size - self_->client_->GetDataSize(), /* exactly_len = */ false)) { + // Failed connection, abort and let somebody retry in the future + defunct_ = true; + self_->Abort(); + guard_.unlock(); throw GenericRpcFailedException(); } } else { @@ -103,7 +121,9 @@ class Client { // Check the response ID. if (res_id != res_type.id && res_id != utils::TypeId::UNKNOWN) { spdlog::error("Message response was of unexpected type"); - self_->client_ = std::nullopt; + // Logically invalid state, connection is still up, defunct stream and release + defunct_ = true; + guard_.unlock(); throw GenericRpcFailedException(); } @@ -112,8 +132,23 @@ class Client { return res_load_(&res_reader); } + bool IsDefunct() const { return defunct_; } + private: + static auto GenBuilderCallback(Client *client, StreamHandler *self) { + return [client, self](const uint8_t *data, size_t size, bool have_more) { + if (self->defunct_) throw GenericRpcFailedException(); + if (!client->client_->Write(data, size, have_more)) { + self->defunct_ = true; + client->Abort(); + self->guard_.unlock(); + throw GenericRpcFailedException(); + } + }; + } + Client *self_; + bool defunct_ = false; std::unique_lock guard_; slk::Builder req_builder_; std::function res_load_; @@ -179,7 +214,7 @@ class Client { TRequestResponse::Request::Save(request, handler.GetBuilder()); // Return the handler to the user. - return std::move(handler); + return handler; } /// Call a previously defined and registered RPC call. This function can diff --git a/src/slk/streams.cpp b/src/slk/streams.cpp index 5125d635a..dc5ef8c3c 100644 --- a/src/slk/streams.cpp +++ b/src/slk/streams.cpp @@ -30,7 +30,7 @@ void Builder::Save(const uint8_t *data, uint64_t size) { to_write = kSegmentMaxDataSize - pos_; } - memcpy(segment_ + sizeof(SegmentSize) + pos_, data + offset, to_write); + memcpy(segment_.data() + sizeof(SegmentSize) + pos_, data + offset, to_write); size -= to_write; pos_ += to_write; @@ -48,15 +48,15 @@ void Builder::FlushSegment(bool final_segment) { size_t total_size = sizeof(SegmentSize) + pos_; SegmentSize size = pos_; - memcpy(segment_, &size, sizeof(SegmentSize)); + memcpy(segment_.data(), &size, sizeof(SegmentSize)); if (final_segment) { SegmentSize footer = 0; - memcpy(segment_ + total_size, &footer, sizeof(SegmentSize)); + memcpy(segment_.data() + total_size, &footer, sizeof(SegmentSize)); total_size += sizeof(SegmentSize); } - write_func_(segment_, total_size, !final_segment); + write_func_(segment_.data(), total_size, !final_segment); pos_ = 0; } diff --git a/src/slk/streams.hpp b/src/slk/streams.hpp index 587b7830b..691189443 100644 --- a/src/slk/streams.hpp +++ b/src/slk/streams.hpp @@ -46,7 +46,11 @@ static_assert(kSegmentMaxDataSize <= std::numeric_limits::max(), /// Builder used to create a SLK segment stream. class Builder { public: - Builder(std::function write_func); + explicit Builder(std::function write_func); + Builder(Builder &&other, std::function write_func) + : write_func_{std::move(write_func)}, pos_{std::exchange(other.pos_, 0)}, segment_{other.segment_} { + other.write_func_ = [](const uint8_t *, size_t, bool) { /* Moved builder is defunct, no write possible */ }; + } /// Function used internally by SLK to serialize the data. void Save(const uint8_t *data, uint64_t size); @@ -59,7 +63,7 @@ class Builder { std::function write_func_; size_t pos_{0}; - uint8_t segment_[kSegmentMaxTotalSize]; + std::array segment_; }; /// Exception that will be thrown if segments can't be decoded from the byte diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 9f6d8d4d7..150a02cc7 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -39,6 +39,7 @@ add_library(mg-storage-v2 STATIC replication/slk.cpp replication/rpc.cpp replication/replication_storage_state.cpp - inmemory/replication/replication_client.cpp + inmemory/replication/recovery.cpp ) + target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory) diff --git a/src/storage/v2/disk/storage.hpp b/src/storage/v2/disk/storage.hpp index 3575e685d..e93566c09 100644 --- a/src/storage/v2/disk/storage.hpp +++ b/src/storage/v2/disk/storage.hpp @@ -313,12 +313,6 @@ class DiskStorage final : public Storage { uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); - auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig & /*config*/, - const memgraph::replication::ReplicationEpoch * /*current_epoch*/) - -> std::unique_ptr override { - throw utils::BasicException("Disk storage mode does not support replication."); - } - std::unique_ptr kvstore_; DurableMetadata durable_metadata_; EdgeImportMode edge_import_status_{EdgeImportMode::INACTIVE}; diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index 1240bd52e..9a43a2876 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -96,6 +96,7 @@ std::vector GetSnapshotFiles(const std::filesystem::path MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); } + std::sort(snapshot_files.begin(), snapshot_files.end()); return snapshot_files; } @@ -106,13 +107,17 @@ std::optional> GetWalFiles(const std::filesystem: std::vector wal_files; std::error_code error_code; + // There could be multiple "current" WAL files, the "_current" tag just means that the previous session didn't + // finalize. We cannot skip based on name, will be able to skip based on invalid data or sequence number, so the + // actual current wal will be skipped for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { if (!item.is_regular_file()) continue; try { auto info = ReadWalInfo(item.path()); - if ((uuid.empty() || info.uuid == uuid) && (!current_seq_num || info.seq_num < *current_seq_num)) + if ((uuid.empty() || info.uuid == uuid) && (!current_seq_num || info.seq_num < *current_seq_num)) { wal_files.emplace_back(info.seq_num, info.from_timestamp, info.to_timestamp, std::move(info.uuid), std::move(info.epoch_id), item.path()); + } } catch (const RecoveryFailure &e) { spdlog::warn("Failed to read {}", item.path()); continue; @@ -120,6 +125,7 @@ std::optional> GetWalFiles(const std::filesystem: } MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); + // Sort based on the sequence number, not the file name std::sort(wal_files.begin(), wal_files.end()); return std::move(wal_files); } @@ -246,8 +252,6 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di std::optional snapshot_timestamp; if (!snapshot_files.empty()) { spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory); - // Order the files by name - std::sort(snapshot_files.begin(), snapshot_files.end()); // UUID used for durability is the UUID of the last snapshot file. *uuid = snapshot_files.back().uuid; diff --git a/src/storage/v2/inmemory/replication/recovery.cpp b/src/storage/v2/inmemory/replication/recovery.cpp new file mode 100644 index 000000000..b62fdc2f9 --- /dev/null +++ b/src/storage/v2/inmemory/replication/recovery.cpp @@ -0,0 +1,238 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/inmemory/replication/recovery.hpp" +#include +#include +#include +#include +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/replication/recovery.hpp" +#include "utils/on_scope_exit.hpp" +#include "utils/variant_helpers.hpp" + +namespace memgraph::storage { + +// Handler for transferring the current WAL file whose data is +// contained in the internal buffer and the file. +class InMemoryCurrentWalHandler { + public: + explicit InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client); + void AppendFilename(const std::string &filename); + + void AppendSize(size_t size); + + void AppendFileData(utils::InputFile *file); + + void AppendBufferData(const uint8_t *buffer, size_t buffer_size); + + /// @throw rpc::RpcFailedException + replication::CurrentWalRes Finalize(); + + private: + rpc::Client::StreamHandler stream_; +}; + +////// CurrentWalHandler ////// +InMemoryCurrentWalHandler::InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client) + : stream_(rpc_client.Stream(storage->id())) {} + +void InMemoryCurrentWalHandler::AppendFilename(const std::string &filename) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteString(filename); +} + +void InMemoryCurrentWalHandler::AppendSize(const size_t size) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteUint(size); +} + +void InMemoryCurrentWalHandler::AppendFileData(utils::InputFile *file) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteFileData(file); +} + +void InMemoryCurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteBuffer(buffer, buffer_size); +} + +replication::CurrentWalRes InMemoryCurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } + +////// ReplicationClient Helpers ////// +replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, + const std::vector &wal_files) { + MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); + auto stream = client.Stream(std::move(db_name), wal_files.size()); + replication::Encoder encoder(stream.GetBuilder()); + for (const auto &wal : wal_files) { + spdlog::debug("Sending wal file: {}", wal); + encoder.WriteFile(wal); + } + return stream.AwaitResponse(); +} + +replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path) { + auto stream = client.Stream(std::move(db_name)); + replication::Encoder encoder(stream.GetBuilder()); + encoder.WriteFile(path); + return stream.AwaitResponse(); +} + +uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file) { + InMemoryCurrentWalHandler stream{storage, client}; + stream.AppendFilename(wal_file.Path().filename()); + utils::InputFile file; + MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file at {}!", wal_file.Path()); + const auto [buffer, buffer_size] = wal_file.CurrentFileBuffer(); + stream.AppendSize(file.GetSize() + buffer_size); + stream.AppendFileData(&file); + stream.AppendBufferData(buffer, buffer_size); + auto response = stream.Finalize(); + return response.current_commit_timestamp; +} + +/// This method tries to find the optimal path for recoverying a single replica. +/// Based on the last commit transfered to replica it tries to update the +/// replica using durability files - WALs and Snapshots. WAL files are much +/// smaller in size as they contain only the Deltas (changes) made during the +/// transactions while Snapshots contain all the data. For that reason we prefer +/// WALs as much as possible. As the WAL file that is currently being updated +/// can change during the process we ignore it as much as possible. Also, it +/// uses the transaction lock so locking it can be really expensive. After we +/// fetch the list of finalized WALs, we try to find the longest chain of +/// sequential WALs, starting from the latest one, that will update the recovery +/// with the all missed updates. If the WAL chain cannot be created, replica is +/// behind by a lot, so we use the regular recovery process, we send the latest +/// snapshot and all the necessary WAL files, starting from the newest WAL that +/// contains a timestamp before the snapshot. If we registered the existence of +/// the current WAL, we add the sequence number we read from it to the recovery +/// process. After all the other steps are finished, if the current WAL contains +/// the same sequence number, it's the same WAL we read while fetching the +/// recovery steps, so we can safely send it to the replica. +/// We assume that the property of preserving at least 1 WAL before the snapshot +/// is satisfied as we extract the timestamp information from it. +std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage) { + std::vector recovery_steps; + auto locker_acc = file_locker->Access(); + + // First check if we can recover using the current wal file only + // otherwise save the seq_num of the current wal file + // This lock is also necessary to force the missed transaction to finish. + std::optional current_wal_seq_num; + std::optional current_wal_from_timestamp; + + std::unique_lock transaction_guard( + storage->engine_lock_); // Hold the storage lock so the current wal file cannot be changed + (void)locker_acc.AddPath(storage->wal_directory_); // Protect all WALs from being deleted + + if (storage->wal_file_) { + current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); + current_wal_from_timestamp.emplace(storage->wal_file_->FromTimestamp()); + // No need to hold the lock since the current WAL is present and we can simply skip them + transaction_guard.unlock(); + } + + // Read in finalized WAL files (excluding the current/active WAL) + utils::OnScopeExit + release_wal_dir( // Each individually used file will be locked, so at the end, the dir can be released + [&locker_acc, &wal_dir = storage->wal_directory_]() { (void)locker_acc.RemovePath(wal_dir); }); + // Get WAL files, ordered by timestamp, from oldest to newest + auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); + MG_ASSERT(wal_files, "Wal files could not be loaded"); + if (transaction_guard.owns_lock()) + transaction_guard.unlock(); // In case we didn't have a current wal file, we can unlock only now since there is no + // guarantee what we'll see after we add the wal file + + // Read in snapshot files + (void)locker_acc.AddPath(storage->snapshot_directory_); // Protect all snapshots from being deleted + utils::OnScopeExit + release_snapshot_dir( // Each individually used file will be locked, so at the end, the dir can be released + [&locker_acc, &snapshot_dir = storage->snapshot_directory_]() { (void)locker_acc.RemovePath(snapshot_dir); }); + auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); + std::optional latest_snapshot{}; + if (!snapshot_files.empty()) { + latest_snapshot.emplace(std::move(snapshot_files.back())); + } + + auto add_snapshot = [&]() { + if (!latest_snapshot) return; + const auto lock_success = locker_acc.AddPath(latest_snapshot->path); + MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant snapshot path."); + recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); + }; + + // Check if we need the snapshot or if the WAL chain is enough + if (!wal_files->empty()) { + // Find WAL chain that contains the replica's commit timestamp + auto wal_chain_it = wal_files->rbegin(); + auto prev_seq{wal_chain_it->seq_num}; + for (; wal_chain_it != wal_files->rend(); ++wal_chain_it) { + if (prev_seq - wal_chain_it->seq_num > 1) { + // Broken chain, must have a snapshot that covers the missing commits + if (wal_chain_it->from_timestamp > replica_commit) { + // Chain does not go far enough, check the snapshot + MG_ASSERT(latest_snapshot, "Missing snapshot, while the WAL chain does not cover enough time."); + // Check for a WAL file that connects the snapshot to the chain + for (;; --wal_chain_it) { + // Going from the newest WAL files, find the first one that has a from_timestamp older than the snapshot + // NOTE: It could be that the only WAL needed is the current one + if (wal_chain_it->from_timestamp <= latest_snapshot->start_timestamp) { + break; + } + if (wal_chain_it == wal_files->rbegin()) break; + } + // Add snapshot to recovery steps + add_snapshot(); + } + break; + } + + if (wal_chain_it->to_timestamp <= replica_commit) { + // Got to a WAL that is older than what we need to recover the replica + break; + } + + prev_seq = wal_chain_it->seq_num; + } + + // Copy and lock the chain part we need, from oldest to newest + RecoveryWals rw{}; + rw.reserve(std::distance(wal_files->rbegin(), wal_chain_it)); + for (auto wal_it = wal_chain_it.base(); wal_it != wal_files->end(); ++wal_it) { + const auto lock_success = locker_acc.AddPath(wal_it->path); + MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant WAL path."); + rw.emplace_back(std::move(wal_it->path)); + } + if (!rw.empty()) { + recovery_steps.emplace_back(std::in_place_type_t{}, std::move(rw)); + } + + } else { + // No WAL chain, check if we need the snapshot + if (!current_wal_from_timestamp || replica_commit < *current_wal_from_timestamp) { + // No current wal or current wal too new + add_snapshot(); + } + } + + // In all cases, if we have a current wal file we need to use itW + if (current_wal_seq_num) { + // NOTE: File not handled directly, so no need to lock it + recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); + } + + return recovery_steps; +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/recovery.hpp b/src/storage/v2/inmemory/replication/recovery.hpp new file mode 100644 index 000000000..2025800ab --- /dev/null +++ b/src/storage/v2/inmemory/replication/recovery.hpp @@ -0,0 +1,32 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/replication/recovery.hpp" +#include "storage/v2/replication/replication_client.hpp" + +namespace memgraph::storage { +class InMemoryStorage; + +////// ReplicationClient Helpers ////// + +replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, + const std::vector &wal_files); + +replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path); + +uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file); + +auto GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage) -> std::vector; + +} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.cpp b/src/storage/v2/inmemory/replication/replication_client.cpp deleted file mode 100644 index b8ecc1c72..000000000 --- a/src/storage/v2/inmemory/replication/replication_client.cpp +++ /dev/null @@ -1,349 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v2/inmemory/replication/replication_client.hpp" - -#include "storage/v2/durability/durability.hpp" -#include "storage/v2/inmemory/storage.hpp" - -namespace memgraph::storage { - -namespace { -template -[[maybe_unused]] inline constexpr bool always_false_v = false; -} // namespace - -// Handler for transfering the current WAL file whose data is -// contained in the internal buffer and the file. -class CurrentWalHandler { - public: - explicit CurrentWalHandler(ReplicationClient *self); - void AppendFilename(const std::string &filename); - - void AppendSize(size_t size); - - void AppendFileData(utils::InputFile *file); - - void AppendBufferData(const uint8_t *buffer, size_t buffer_size); - - /// @throw rpc::RpcFailedException - replication::CurrentWalRes Finalize(); - - private: - ReplicationClient *self_; - rpc::Client::StreamHandler stream_; -}; - -////// CurrentWalHandler ////// -CurrentWalHandler::CurrentWalHandler(ReplicationClient *self) - : self_(self), stream_(self_->rpc_client_.Stream(self->GetStorageId())) {} - -void CurrentWalHandler::AppendFilename(const std::string &filename) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteString(filename); -} - -void CurrentWalHandler::AppendSize(const size_t size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteUint(size); -} - -void CurrentWalHandler::AppendFileData(utils::InputFile *file) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteFileData(file); -} - -void CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteBuffer(buffer, buffer_size); -} - -replication::CurrentWalRes CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } - -////// ReplicationClient Helpers ////// - -replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, - const std::vector &wal_files) { - MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); - auto stream = client.Stream(std::move(db_name), wal_files.size()); - replication::Encoder encoder(stream.GetBuilder()); - for (const auto &wal : wal_files) { - spdlog::debug("Sending wal file: {}", wal); - encoder.WriteFile(wal); - } - return stream.AwaitResponse(); -} - -replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path) { - auto stream = client.Stream(std::move(db_name)); - replication::Encoder encoder(stream.GetBuilder()); - encoder.WriteFile(path); - return stream.AwaitResponse(); -} - -uint64_t ReplicateCurrentWal(CurrentWalHandler &stream, durability::WalFile const &wal_file) { - stream.AppendFilename(wal_file.Path().filename()); - utils::InputFile file; - MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file!"); - const auto [buffer, buffer_size] = wal_file.CurrentFileBuffer(); - stream.AppendSize(file.GetSize() + buffer_size); - stream.AppendFileData(&file); - stream.AppendBufferData(buffer, buffer_size); - auto response = stream.Finalize(); - return response.current_commit_timestamp; -} - -////// ReplicationClient ////// - -InMemoryReplicationClient::InMemoryReplicationClient(InMemoryStorage *storage, - const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch) - : ReplicationClient{storage, config, epoch} {} - -void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { - spdlog::debug("Starting replica recover"); - auto *storage = static_cast(storage_); - while (true) { - auto file_locker = storage->file_retainer_.AddLocker(); - - const auto steps = GetRecoverySteps(replica_commit, &file_locker); - int i = 0; - for (const InMemoryReplicationClient::RecoveryStep &recovery_step : steps) { - spdlog::trace("Recovering in step: {}", i++); - try { - std::visit( - [&, this](T &&arg) { - using StepType = std::remove_cvref_t; - if constexpr (std::is_same_v) { // TODO: split into 3 overloads - spdlog::debug("Sending the latest snapshot file: {}", arg); - auto response = TransferSnapshot(storage->id(), rpc_client_, arg); - replica_commit = response.current_commit_timestamp; - } else if constexpr (std::is_same_v) { - spdlog::debug("Sending the latest wal files"); - auto response = TransferWalFiles(storage->id(), rpc_client_, arg); - replica_commit = response.current_commit_timestamp; - spdlog::debug("Wal files successfully transferred."); - } else if constexpr (std::is_same_v) { - std::unique_lock transaction_guard(storage->engine_lock_); - if (storage->wal_file_ && storage->wal_file_->SequenceNumber() == arg.current_wal_seq_num) { - storage->wal_file_->DisableFlushing(); - transaction_guard.unlock(); - spdlog::debug("Sending current wal file"); - auto streamHandler = CurrentWalHandler{this}; - replica_commit = ReplicateCurrentWal(streamHandler, *storage->wal_file_); - storage->wal_file_->EnableFlushing(); - } else { - spdlog::debug("Cannot recover using current wal file"); - } - } else { - static_assert(always_false_v, "Missing type from variant visitor"); - } - }, - recovery_step); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - return; - } - } - - spdlog::trace("Current timestamp on replica: {}", replica_commit); - // To avoid the situation where we read a correct commit timestamp in - // one thread, and after that another thread commits a different a - // transaction and THEN we set the state to READY in the first thread, - // we set this lock before checking the timestamp. - // We will detect that the state is invalid during the next commit, - // because replication::AppendDeltasRpc sends the last commit timestamp which - // replica checks if it's the same last commit timestamp it received - // and we will go to recovery. - // By adding this lock, we can avoid that, and go to RECOVERY immediately. - std::unique_lock client_guard{client_lock_}; - const auto last_commit_timestamp = LastCommitTimestamp(); - SPDLOG_INFO("Replica timestamp: {}", replica_commit); - SPDLOG_INFO("Last commit: {}", last_commit_timestamp); - if (last_commit_timestamp == replica_commit) { - replica_state_.store(replication::ReplicaState::READY); - return; - } - } -} - -/// This method tries to find the optimal path for recoverying a single replica. -/// Based on the last commit transfered to replica it tries to update the -/// replica using durability files - WALs and Snapshots. WAL files are much -/// smaller in size as they contain only the Deltas (changes) made during the -/// transactions while Snapshots contain all the data. For that reason we prefer -/// WALs as much as possible. As the WAL file that is currently being updated -/// can change during the process we ignore it as much as possible. Also, it -/// uses the transaction lock so locking it can be really expensive. After we -/// fetch the list of finalized WALs, we try to find the longest chain of -/// sequential WALs, starting from the latest one, that will update the recovery -/// with the all missed updates. If the WAL chain cannot be created, replica is -/// behind by a lot, so we use the regular recovery process, we send the latest -/// snapshot and all the necessary WAL files, starting from the newest WAL that -/// contains a timestamp before the snapshot. If we registered the existence of -/// the current WAL, we add the sequence number we read from it to the recovery -/// process. After all the other steps are finished, if the current WAL contains -/// the same sequence number, it's the same WAL we read while fetching the -/// recovery steps, so we can safely send it to the replica. -/// We assume that the property of preserving at least 1 WAL before the snapshot -/// is satisfied as we extract the timestamp information from it. -std::vector InMemoryReplicationClient::GetRecoverySteps( - const uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker) { - // First check if we can recover using the current wal file only - // otherwise save the seq_num of the current wal file - // This lock is also necessary to force the missed transaction to finish. - std::optional current_wal_seq_num; - std::optional current_wal_from_timestamp; - auto *storage = static_cast(storage_); - if (std::unique_lock transtacion_guard(storage->engine_lock_); storage->wal_file_) { - current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); - current_wal_from_timestamp.emplace(storage->wal_file_->FromTimestamp()); - } - - auto locker_acc = file_locker->Access(); - auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); - MG_ASSERT(wal_files, "Wal files could not be loaded"); - - auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); - std::optional latest_snapshot; - if (!snapshot_files.empty()) { - std::sort(snapshot_files.begin(), snapshot_files.end()); - latest_snapshot.emplace(std::move(snapshot_files.back())); - } - - std::vector recovery_steps; - - // No finalized WAL files were found. This means the difference is contained - // inside the current WAL or the snapshot. - if (wal_files->empty()) { - if (current_wal_from_timestamp && replica_commit >= *current_wal_from_timestamp) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Without the finalized WAL containing the current timestamp of replica, - // we cannot know if the difference is only in the current WAL or we need - // to send the snapshot. - if (latest_snapshot) { - const auto lock_success = locker_acc.AddPath(latest_snapshot->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - } - // if there are no finalized WAL files, snapshot left the current WAL - // as the WAL file containing a transaction before snapshot creation - // so we can be sure that the current WAL is present - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Find the longest chain of WALs for recovery. - // The chain consists ONLY of sequential WALs. - auto rwal_it = wal_files->rbegin(); - - // if the last finalized WAL is before the replica commit - // then we can recovery only from current WAL - if (rwal_it->to_timestamp <= replica_commit) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - uint64_t previous_seq_num{rwal_it->seq_num}; - for (; rwal_it != wal_files->rend(); ++rwal_it) { - // If the difference between two consecutive wal files is not 0 or 1 - // we have a missing WAL in our chain - if (previous_seq_num - rwal_it->seq_num > 1) { - break; - } - - // Find first WAL that contains up to replica commit, i.e. WAL - // that is before the replica commit or conatins the replica commit - // as the last committed transaction OR we managed to find the first WAL - // file. - if (replica_commit >= rwal_it->from_timestamp || rwal_it->seq_num == 0) { - if (replica_commit >= rwal_it->to_timestamp) { - // We want the WAL after because the replica already contains all the - // commits from this WAL - --rwal_it; - } - std::vector wal_chain; - auto distance_from_first = std::distance(rwal_it, wal_files->rend() - 1); - // We have managed to create WAL chain - // We need to lock these files and add them to the chain - for (auto result_wal_it = wal_files->begin() + distance_from_first; result_wal_it != wal_files->end(); - ++result_wal_it) { - const auto lock_success = locker_acc.AddPath(result_wal_it->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - wal_chain.push_back(std::move(result_wal_it->path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(wal_chain)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - return recovery_steps; - } - - previous_seq_num = rwal_it->seq_num; - } - - MG_ASSERT(latest_snapshot, "Invalid durability state, missing snapshot"); - // We didn't manage to find a WAL chain, we need to send the latest snapshot - // with its WALs - const auto lock_success = locker_acc.AddPath(latest_snapshot->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - - std::vector recovery_wal_files; - auto wal_it = wal_files->begin(); - for (; wal_it != wal_files->end(); ++wal_it) { - // Assuming recovery process is correct the snashpot should - // always retain a single WAL that contains a transaction - // before its creation - if (latest_snapshot->start_timestamp < wal_it->to_timestamp) { - if (latest_snapshot->start_timestamp < wal_it->from_timestamp) { - MG_ASSERT(wal_it != wal_files->begin(), "Invalid durability files state"); - --wal_it; - } - break; - } - } - - for (; wal_it != wal_files->end(); ++wal_it) { - const auto lock_success = locker_acc.AddPath(wal_it->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_wal_files.push_back(std::move(wal_it->path)); - } - - // We only have a WAL before the snapshot - if (recovery_wal_files.empty()) { - const auto lock_success = locker_acc.AddPath(wal_files->back().path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_wal_files.push_back(std::move(wal_files->back().path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(recovery_wal_files)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - - return recovery_steps; -} - -} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.hpp b/src/storage/v2/inmemory/replication/replication_client.hpp deleted file mode 100644 index e956838e7..000000000 --- a/src/storage/v2/inmemory/replication/replication_client.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. -#pragma once - -#include "storage/v2/replication/replication_client.hpp" - -namespace memgraph::storage { - -class InMemoryStorage; - -class InMemoryReplicationClient : public ReplicationClient { - public: - InMemoryReplicationClient(InMemoryStorage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch); - - protected: - void RecoverReplica(uint64_t replica_commit) override; - - // TODO: move the GetRecoverySteps stuff below as an internal detail - using RecoverySnapshot = std::filesystem::path; - using RecoveryWals = std::vector; - struct RecoveryCurrentWal { - explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num) : current_wal_seq_num(current_wal_seq_num) {} - uint64_t current_wal_seq_num; - }; - using RecoveryStep = std::variant; - std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker); -}; - -} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 9f00081f6..c27018d24 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -18,7 +18,7 @@ /// REPLICATION /// #include "dbms/inmemory/replication_handlers.hpp" -#include "storage/v2/inmemory/replication/replication_client.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" #include "utils/resource_lock.hpp" #include "utils/stat.hpp" @@ -1608,7 +1608,7 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction // A single transaction will always be contained in a single WAL file. auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire); - repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber(), this); auto append_deltas = [&](auto callback) { // Helper lambda that traverses the delta chain on order to find the first @@ -1767,7 +1767,7 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp, this); } bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, uint64_t final_commit_timestamp) { @@ -1775,7 +1775,7 @@ bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, return true; } - repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber(), this); for (const auto &md_delta : transaction.md_deltas) { switch (md_delta.action) { @@ -1846,7 +1846,7 @@ bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp, this); } void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, @@ -1972,12 +1972,6 @@ utils::FileRetainer::FileLockerAccessor::ret_type InMemoryStorage::UnlockPath() return true; } -auto InMemoryStorage::CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr { - return std::make_unique(this, config, current_epoch); -} - std::unique_ptr InMemoryStorage::Access(std::optional override_isolation_level, bool is_main) { return std::unique_ptr(new InMemoryAccessor{Storage::Accessor::shared_access, this, diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index bfb445332..3c50326b2 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -18,10 +18,13 @@ #include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/label_index.hpp" #include "storage/v2/inmemory/label_property_index.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" +#include "storage/v2/replication/replication_client.hpp" #include "storage/v2/storage.hpp" /// REPLICATION /// #include "replication/config.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" #include "storage/v2/replication/enums.hpp" #include "storage/v2/replication/replication_storage_state.hpp" #include "storage/v2/replication/rpc.hpp" @@ -44,7 +47,10 @@ namespace memgraph::storage { class InMemoryStorage final : public Storage { friend class memgraph::dbms::InMemoryReplicationHandlers; - friend class InMemoryReplicationClient; + friend class ReplicationStorageClient; + friend std::vector GetRecoverySteps(uint64_t replica_commit, + utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage); public: enum class CreateSnapshotError : uint8_t { DisabledForReplica, ReachedMaxNumTries }; @@ -332,10 +338,6 @@ class InMemoryStorage final : public Storage { using Storage::CreateTransaction; Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode, bool is_main) override; - auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr override; - void SetStorageMode(StorageMode storage_mode); private: diff --git a/src/storage/v2/replication/enums.hpp b/src/storage/v2/replication/enums.hpp index e89a1fdd3..be16ca192 100644 --- a/src/storage/v2/replication/enums.hpp +++ b/src/storage/v2/replication/enums.hpp @@ -14,6 +14,6 @@ namespace memgraph::storage::replication { -enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID }; +enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, MAYBE_BEHIND }; } // namespace memgraph::storage::replication diff --git a/src/storage/v2/replication/recovery.hpp b/src/storage/v2/replication/recovery.hpp new file mode 100644 index 000000000..346e03ecd --- /dev/null +++ b/src/storage/v2/replication/recovery.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +namespace memgraph::storage { + +using RecoverySnapshot = std::filesystem::path; +using RecoveryWals = std::vector; +struct RecoveryCurrentWal { + explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num) : current_wal_seq_num(current_wal_seq_num) {} + uint64_t current_wal_seq_num; +}; +using RecoveryStep = std::variant; + +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_client.cpp b/src/storage/v2/replication/replication_client.cpp index 33313b130..3bc1b3d32 100644 --- a/src/storage/v2/replication/replication_client.cpp +++ b/src/storage/v2/replication/replication_client.cpp @@ -9,53 +9,37 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -#include "storage/v2/replication/replication_client.hpp" +#include "replication/replication_client.hpp" +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/storage.hpp" +#include "utils/exceptions.hpp" +#include "utils/variant_helpers.hpp" #include #include -#include "storage/v2/durability/durability.hpp" -#include "storage/v2/storage.hpp" +namespace { +template +[[maybe_unused]] inline constexpr bool always_false_v = false; +} // namespace namespace memgraph::storage { -static auto CreateClientContext(const memgraph::replication::ReplicationClientConfig &config) - -> communication::ClientContext { - return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file} - : communication::ClientContext{}; -} +ReplicationStorageClient::ReplicationStorageClient(::memgraph::replication::ReplicationClient &client) + : client_{client} {} -ReplicationClient::ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch) - : name_{config.name}, - rpc_context_{CreateClientContext(config)}, - rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port), - &rpc_context_}, - replica_check_frequency_{config.replica_check_frequency}, - mode_{config.mode}, - storage_{storage}, - repl_epoch_{epoch} {} - -ReplicationClient::~ReplicationClient() { - auto endpoint = rpc_client_.Endpoint(); - spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port); - thread_pool_.Shutdown(); -} - -uint64_t ReplicationClient::LastCommitTimestamp() const { - return storage_->repl_storage_state_.last_commit_timestamp_.load(); -} - -void ReplicationClient::InitializeClient() { +void ReplicationStorageClient::CheckReplicaState(Storage *storage) { uint64_t current_commit_timestamp{kTimestampInitialId}; - auto stream{rpc_client_.Stream( - storage_->id(), storage_->repl_storage_state_.last_commit_timestamp_, std::string{repl_epoch_->id()})}; + auto &replStorageState = storage->repl_storage_state_; + auto stream{client_.rpc_client_.Stream( + storage->id(), replStorageState.last_commit_timestamp_, std::string{replStorageState.epoch_.id()})}; const auto replica = stream.AwaitResponse(); std::optional branching_point; - if (replica.epoch_id != repl_epoch_->id() && replica.current_commit_timestamp != kTimestampInitialId) { - auto const &history = storage_->repl_storage_state_.history; + if (replica.epoch_id != replStorageState.epoch_.id() && replica.current_commit_timestamp != kTimestampInitialId) { + auto const &history = replStorageState.history; const auto epoch_info_iter = std::find_if(history.crbegin(), history.crend(), [&](const auto &main_epoch_info) { return main_epoch_info.first == replica.epoch_id; }); @@ -71,94 +55,86 @@ void ReplicationClient::InitializeClient() { "Replica {} acted as the Main instance. Both the Main and Replica {} " "now hold unique data. Please resolve data conflicts and start the " "replication on a clean instance.", - name_, name_, name_); + client_.name_, client_.name_, client_.name_); + // State not updated, hence in MAYBE_BEHIND state return; } current_commit_timestamp = replica.current_commit_timestamp; - spdlog::trace("Current timestamp on replica {}: {}", name_, current_commit_timestamp); - spdlog::trace("Current timestamp on main: {}", storage_->repl_storage_state_.last_commit_timestamp_.load()); - if (current_commit_timestamp == storage_->repl_storage_state_.last_commit_timestamp_.load()) { - spdlog::debug("Replica '{}' up to date", name_); - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::READY); - } else { - spdlog::debug("Replica '{}' is behind", name_); - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::RECOVERY); + spdlog::trace("Current timestamp on replica {}: {}", client_.name_, current_commit_timestamp); + spdlog::trace("Current timestamp on main: {}", replStorageState.last_commit_timestamp_.load()); + replica_state_.WithLock([&](auto &state) { + if (current_commit_timestamp == replStorageState.last_commit_timestamp_.load()) { + spdlog::debug("Replica '{}' up to date", client_.name_); + state = replication::ReplicaState::READY; + } else { + spdlog::debug("Replica '{}' is behind", client_.name_); + state = replication::ReplicaState::RECOVERY; + client_.thread_pool_.AddTask( + [storage, current_commit_timestamp, this] { this->RecoverReplica(current_commit_timestamp, storage); }); } - thread_pool_.AddTask([=, this] { this->RecoverReplica(current_commit_timestamp); }); - } + }); } -TimestampInfo ReplicationClient::GetTimestampInfo() { +TimestampInfo ReplicationStorageClient::GetTimestampInfo(Storage const *storage) { TimestampInfo info; info.current_timestamp_of_replica = 0; info.current_number_of_timestamp_behind_master = 0; try { - auto stream{rpc_client_.Stream(storage_->id())}; + auto stream{client_.rpc_client_.Stream(storage->id())}; const auto response = stream.AwaitResponse(); const auto is_success = response.success; if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); } - auto main_time_stamp = storage_->repl_storage_state_.last_commit_timestamp_.load(); + auto main_time_stamp = storage->repl_storage_state_.last_commit_timestamp_.load(); info.current_timestamp_of_replica = response.current_commit_timestamp; info.current_number_of_timestamp_behind_master = response.current_commit_timestamp - main_time_stamp; } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably won't - // block + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably + // won't block } return info; } -void ReplicationClient::HandleRpcFailure() { - spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication")); - TryInitializeClientAsync(); +void ReplicationStorageClient::LogRpcFailure() { + spdlog::error( + utils::MessageWithLink("Couldn't replicate data to {}.", client_.name_, "https://memgr.ph/replication")); } -void ReplicationClient::TryInitializeClientAsync() { - thread_pool_.AddTask([this] { - rpc_client_.Abort(); - this->TryInitializeClientSync(); - }); +void ReplicationStorageClient::TryCheckReplicaStateAsync(Storage *storage) { + client_.thread_pool_.AddTask([storage, this] { this->TryCheckReplicaStateSync(storage); }); } -void ReplicationClient::TryInitializeClientSync() { +void ReplicationStorageClient::TryCheckReplicaStateSync(Storage *storage) { try { - InitializeClient(); + CheckReplicaState(storage); } catch (const rpc::VersionMismatchRpcFailedException &) { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); spdlog::error( utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}. Because the replica " "deployed is not a compatible version.", - name_, rpc_client_.Endpoint(), "https://memgr.ph/replication")); + client_.name_, client_.rpc_client_.Endpoint(), "https://memgr.ph/replication")); } catch (const rpc::RpcFailedException &) { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_, - rpc_client_.Endpoint(), "https://memgr.ph/replication")); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", client_.name_, + client_.rpc_client_.Endpoint(), "https://memgr.ph/replication")); } } -void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) { - std::unique_lock guard(client_lock_); - const auto status = replica_state_.load(); - switch (status) { - case replication::ReplicaState::RECOVERY: - spdlog::debug("Replica {} is behind MAIN instance", name_); +void ReplicationStorageClient::StartTransactionReplication(const uint64_t current_wal_seq_num, Storage *storage) { + auto locked_state = replica_state_.Lock(); + switch (*locked_state) { + using enum replication::ReplicaState; + case RECOVERY: + spdlog::debug("Replica {} is behind MAIN instance", client_.name_); return; - case replication::ReplicaState::REPLICATING: - spdlog::debug("Replica {} missed a transaction", name_); + case REPLICATING: + spdlog::debug("Replica {} missed a transaction", client_.name_); // We missed a transaction because we're still replicating // the previous transaction so we need to go to RECOVERY // state to catch up with the missing transaction @@ -166,143 +142,169 @@ void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_s // an error can happen while we're replicating the previous // transaction after which the client should go to // INVALID state before starting the recovery process - replica_state_.store(replication::ReplicaState::RECOVERY); + // + // This is a signal to any async streams that are still finalizing to start recovery, since this commit will be + // missed. + *locked_state = RECOVERY; return; - case replication::ReplicaState::INVALID: - HandleRpcFailure(); + case MAYBE_BEHIND: + spdlog::error( + utils::MessageWithLink("Couldn't replicate data to {}.", client_.name_, "https://memgr.ph/replication")); + TryCheckReplicaStateAsync(storage); return; - case replication::ReplicaState::READY: + case READY: MG_ASSERT(!replica_stream_); try { - replica_stream_.emplace( - ReplicaStream{this, storage_->repl_storage_state_.last_commit_timestamp_.load(), current_wal_seq_num}); - replica_state_.store(replication::ReplicaState::REPLICATING); + replica_stream_.emplace(storage, client_.rpc_client_, current_wal_seq_num); + *locked_state = REPLICATING; } catch (const rpc::RpcFailedException &) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); + *locked_state = MAYBE_BEHIND; + LogRpcFailure(); } return; } } -bool ReplicationClient::FinalizeTransactionReplication() { +bool ReplicationStorageClient::FinalizeTransactionReplication(Storage *storage) { // We can only check the state because it guarantees to be only // valid during a single transaction replication (if the assumption // that this and other transaction replication functions can only be // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { + if (State() != replication::ReplicaState::REPLICATING) { return false; } - auto task = [this]() { + if (replica_stream_->IsDefunct()) return false; + + auto task = [storage, this]() { MG_ASSERT(replica_stream_, "Missing stream for transaction deltas"); try { auto response = replica_stream_->Finalize(); - replica_stream_.reset(); - std::unique_lock client_guard(client_lock_); - if (!response.success || replica_state_ == replication::ReplicaState::RECOVERY) { - replica_state_.store(replication::ReplicaState::RECOVERY); - thread_pool_.AddTask([&, this] { this->RecoverReplica(response.current_commit_timestamp); }); - } else { - replica_state_.store(replication::ReplicaState::READY); + return replica_state_.WithLock([storage, &response, this](auto &state) { + replica_stream_.reset(); + if (!response.success || state == replication::ReplicaState::RECOVERY) { + state = replication::ReplicaState::RECOVERY; + client_.thread_pool_.AddTask( + [storage, &response, this] { this->RecoverReplica(response.current_commit_timestamp, storage); }); + return false; + } + state = replication::ReplicaState::READY; return true; - } + }); } catch (const rpc::RpcFailedException &) { - replica_stream_.reset(); - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); + replica_state_.WithLock([this](auto &state) { + replica_stream_.reset(); + state = replication::ReplicaState::MAYBE_BEHIND; + }); + LogRpcFailure(); + return false; } - return false; }; - if (mode_ == memgraph::replication::ReplicationMode::ASYNC) { - thread_pool_.AddTask([=] { (void)task(); }); + if (client_.mode_ == memgraph::replication::ReplicationMode::ASYNC) { + client_.thread_pool_.AddTask([task = std::move(task)] { (void)task(); }); return true; } return task(); } -void ReplicationClient::FrequentCheck() { - const auto is_success = std::invoke([this]() { - try { - auto stream{rpc_client_.Stream()}; - const auto response = stream.AwaitResponse(); - return response.success; - } catch (const rpc::RpcFailedException &) { - return false; - } - }); - // States: READY, REPLICATING, RECOVERY, INVALID - // If success && ready, replicating, recovery -> stay the same because something good is going on. - // If success && INVALID -> [it's possible that replica came back to life] -> TryInitializeClient. - // If fail -> [replica is not reachable at all] -> INVALID state. - // NOTE: TryInitializeClient might return nothing if there is a branching point. - // NOTE: The early return pattern simplified the code, but the behavior should be as explained. - if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - return; - } - if (replica_state_.load() == replication::ReplicaState::INVALID) { - TryInitializeClientAsync(); - } +void ReplicationStorageClient::Start(Storage *storage) { + spdlog::trace("Replication client started for database \"{}\"", storage->id()); + TryCheckReplicaStateSync(storage); } -void ReplicationClient::Start() { - auto const &endpoint = rpc_client_.Endpoint(); - spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); - - TryInitializeClientSync(); - - // Help the user to get the most accurate replica state possible. - if (replica_check_frequency_ > std::chrono::seconds(0)) { - replica_checker_.Run("Replica Checker", replica_check_frequency_, [this] { this->FrequentCheck(); }); +void ReplicationStorageClient::RecoverReplica(uint64_t replica_commit, memgraph::storage::Storage *storage) { + if (storage->storage_mode_ != StorageMode::IN_MEMORY_TRANSACTIONAL) { + throw utils::BasicException("Only InMemoryTransactional mode supports replication!"); } -} + spdlog::debug("Starting replica recovery"); + auto *mem_storage = static_cast(storage); -void ReplicationClient::IfStreamingTransaction(const std::function &callback) { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return; - } + while (true) { + auto file_locker = mem_storage->file_retainer_.AddLocker(); - try { - callback(*replica_stream_); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); + const auto steps = GetRecoverySteps(replica_commit, &file_locker, mem_storage); + int i = 0; + for (const RecoveryStep &recovery_step : steps) { + spdlog::trace("Recovering in step: {}", i++); + try { + rpc::Client &rpcClient = client_.rpc_client_; + std::visit(utils::Overloaded{ + [&replica_commit, mem_storage, &rpcClient](RecoverySnapshot const &snapshot) { + spdlog::debug("Sending the latest snapshot file: {}", snapshot); + auto response = TransferSnapshot(mem_storage->id(), rpcClient, snapshot); + replica_commit = response.current_commit_timestamp; + }, + [&replica_commit, mem_storage, &rpcClient](RecoveryWals const &wals) { + spdlog::debug("Sending the latest wal files"); + auto response = TransferWalFiles(mem_storage->id(), rpcClient, wals); + replica_commit = response.current_commit_timestamp; + spdlog::debug("Wal files successfully transferred."); + }, + [&replica_commit, mem_storage, &rpcClient](RecoveryCurrentWal const ¤t_wal) { + std::unique_lock transaction_guard(mem_storage->engine_lock_); + if (mem_storage->wal_file_ && + mem_storage->wal_file_->SequenceNumber() == current_wal.current_wal_seq_num) { + mem_storage->wal_file_->DisableFlushing(); + transaction_guard.unlock(); + spdlog::debug("Sending current wal file"); + replica_commit = ReplicateCurrentWal(mem_storage, rpcClient, *mem_storage->wal_file_); + mem_storage->wal_file_->EnableFlushing(); + } else { + spdlog::debug("Cannot recover using current wal file"); + } + }, + [](auto const &in) { + static_assert(always_false_v, "Missing type from variant visitor"); + }, + }, + recovery_step); + } catch (const rpc::RpcFailedException &) { + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); + return; + } + } + + spdlog::trace("Current timestamp on replica: {}", replica_commit); + // To avoid the situation where we read a correct commit timestamp in + // one thread, and after that another thread commits a different a + // transaction and THEN we set the state to READY in the first thread, + // we set this lock before checking the timestamp. + // We will detect that the state is invalid during the next commit, + // because replication::AppendDeltasRpc sends the last commit timestamp which + // replica checks if it's the same last commit timestamp it received + // and we will go to recovery. + // By adding this lock, we can avoid that, and go to RECOVERY immediately. + const auto last_commit_timestamp = storage->repl_storage_state_.last_commit_timestamp_.load(); + SPDLOG_INFO("Replica timestamp: {}", replica_commit); + SPDLOG_INFO("Last commit: {}", last_commit_timestamp); + if (last_commit_timestamp == replica_commit) { + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::READY; }); + return; } - HandleRpcFailure(); } } ////// ReplicaStream ////// -ReplicaStream::ReplicaStream(ReplicationClient *self, const uint64_t previous_commit_timestamp, - const uint64_t current_seq_num) - : self_(self), - stream_(self_->rpc_client_.Stream(self->GetStorageId(), previous_commit_timestamp, - current_seq_num)) { +ReplicaStream::ReplicaStream(Storage *storage, rpc::Client &rpc_client, const uint64_t current_seq_num) + : storage_{storage}, + stream_(rpc_client.Stream( + storage->id(), storage->repl_storage_state_.last_commit_timestamp_.load(), current_seq_num)) { replication::Encoder encoder{stream_.GetBuilder()}; - - encoder.WriteString(self->repl_epoch_->id()); + encoder.WriteString(storage->repl_storage_state_.epoch_.id()); } void ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - auto *storage = self_->GetStorage(); - EncodeDelta(&encoder, storage->name_id_mapper_.get(), storage->config_.items, delta, vertex, final_commit_timestamp); + EncodeDelta(&encoder, storage_->name_id_mapper_.get(), storage_->config_.items, delta, vertex, + final_commit_timestamp); } void ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge, uint64_t final_commit_timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - EncodeDelta(&encoder, self_->GetStorage()->name_id_mapper_.get(), delta, edge, final_commit_timestamp); + EncodeDelta(&encoder, storage_->name_id_mapper_.get(), delta, edge, final_commit_timestamp); } void ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) { @@ -314,11 +316,10 @@ void ReplicaStream::AppendOperation(durability::StorageMetadataOperation operati const std::set &properties, const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - EncodeOperation(&encoder, self_->GetStorage()->name_id_mapper_.get(), operation, label, properties, stats, - property_stats, timestamp); + EncodeOperation(&encoder, storage_->name_id_mapper_.get(), operation, label, properties, stats, property_stats, + timestamp); } replication::AppendDeltasRes ReplicaStream::Finalize() { return stream_.AwaitResponse(); } -auto ReplicationClient::GetStorageId() const -> std::string { return storage_->id(); } } // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index 8cd8cb384..3d2c019e9 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -13,6 +13,8 @@ #include "replication/config.hpp" #include "replication/epoch.hpp" +#include "replication/messages.hpp" +#include "replication/replication_client.hpp" #include "rpc/client.hpp" #include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/id_types.hpp" @@ -23,9 +25,12 @@ #include "storage/v2/replication/rpc.hpp" #include "utils/file_locker.hpp" #include "utils/scheduler.hpp" +#include "utils/synchronized.hpp" #include "utils/thread_pool.hpp" #include +#include +#include #include #include #include @@ -37,12 +42,12 @@ struct Delta; struct Vertex; struct Edge; class Storage; -class ReplicationClient; +class ReplicationStorageClient; // Handler used for transferring the current transaction. class ReplicaStream { public: - explicit ReplicaStream(ReplicationClient *self, uint64_t previous_commit_timestamp, uint64_t current_seq_num); + explicit ReplicaStream(Storage *storage, rpc::Client &rpc_client, uint64_t current_seq_num); /// @throw rpc::RpcFailedException void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp); @@ -61,85 +66,84 @@ class ReplicaStream { /// @throw rpc::RpcFailedException replication::AppendDeltasRes Finalize(); + bool IsDefunct() const { return stream_.IsDefunct(); } + private: - ReplicationClient *self_; + Storage *storage_; rpc::Client::StreamHandler stream_; }; -class ReplicationClient { - friend class CurrentWalHandler; +template +concept InvocableWithStream = std::invocable; + +// TODO Rename to something without the word "client" +class ReplicationStorageClient { + friend class InMemoryCurrentWalHandler; friend class ReplicaStream; + friend struct ::memgraph::replication::ReplicationClient; public: - ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch); + explicit ReplicationStorageClient(::memgraph::replication::ReplicationClient &client); - ReplicationClient(ReplicationClient const &) = delete; - ReplicationClient &operator=(ReplicationClient const &) = delete; - ReplicationClient(ReplicationClient &&) noexcept = delete; - ReplicationClient &operator=(ReplicationClient &&) noexcept = delete; + ReplicationStorageClient(ReplicationStorageClient const &) = delete; + ReplicationStorageClient &operator=(ReplicationStorageClient const &) = delete; + ReplicationStorageClient(ReplicationStorageClient &&) noexcept = delete; + ReplicationStorageClient &operator=(ReplicationStorageClient &&) noexcept = delete; - virtual ~ReplicationClient(); + ~ReplicationStorageClient() = default; - auto Mode() const -> memgraph::replication::ReplicationMode { return mode_; } - auto Name() const -> std::string const & { return name_; } - auto Endpoint() const -> io::network::Endpoint const & { return rpc_client_.Endpoint(); } - auto State() const -> replication::ReplicaState { return replica_state_.load(); } - auto GetTimestampInfo() -> TimestampInfo; + // TODO Remove the client related functions + auto Mode() const -> memgraph::replication::ReplicationMode { return client_.mode_; } + auto Name() const -> std::string const & { return client_.name_; } + auto Endpoint() const -> io::network::Endpoint const & { return client_.rpc_client_.Endpoint(); } - auto GetStorageId() const -> std::string; + auto State() const -> replication::ReplicaState { return replica_state_.WithLock(std::identity()); } + auto GetTimestampInfo(Storage const *storage) -> TimestampInfo; + + void Start(Storage *storage); + void StartTransactionReplication(uint64_t current_wal_seq_num, Storage *storage); - void Start(); - void StartTransactionReplication(uint64_t current_wal_seq_num); // Replication clients can be removed at any point // so to avoid any complexity of checking if the client was removed whenever // we want to send part of transaction and to avoid adding some GC logic this // function will run a callback if, after previously callling // StartTransactionReplication, stream is created. - void IfStreamingTransaction(const std::function &callback); + template + void IfStreamingTransaction(F &&callback) { + // We can only check the state because it guarantees to be only + // valid during a single transaction replication (if the assumption + // that this and other transaction replication functions can only be + // called from a one thread stands) + if (State() != replication::ReplicaState::REPLICATING) { + return; + } + if (replica_stream_->IsDefunct()) return; + try { + callback(*replica_stream_); // failure state what if not streaming (std::nullopt) + } catch (const rpc::RpcFailedException &) { + return replica_state_.WithLock([](auto &state) { state = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); + } + } + // Return whether the transaction could be finalized on the replication client or not. - [[nodiscard]] bool FinalizeTransactionReplication(); + [[nodiscard]] bool FinalizeTransactionReplication(Storage *storage); - protected: - virtual void RecoverReplica(uint64_t replica_commit) = 0; + void TryCheckReplicaStateAsync(Storage *storage); // TODO Move back to private + private: + void RecoverReplica(uint64_t replica_commit, memgraph::storage::Storage *storage); - auto GetStorage() -> Storage * { return storage_; } - auto LastCommitTimestamp() const -> uint64_t; - void InitializeClient(); - void HandleRpcFailure(); - void TryInitializeClientAsync(); - void TryInitializeClientSync(); - void FrequentCheck(); + void CheckReplicaState(Storage *storage); + void LogRpcFailure(); + void TryCheckReplicaStateSync(Storage *storage); + void FrequentCheck(Storage *storage); - std::string name_; - communication::ClientContext rpc_context_; - rpc::Client rpc_client_; - std::chrono::seconds replica_check_frequency_; - - std::optional replica_stream_; - memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC}; - - utils::SpinLock client_lock_; - // This thread pool is used for background tasks so we don't - // block the main storage thread - // We use only 1 thread for 2 reasons: - // - background tasks ALWAYS contain some kind of RPC communication. - // We can't have multiple RPC communication from a same client - // because that's not logically valid (e.g. you cannot send a snapshot - // and WAL at a same time because WAL will arrive earlier and be applied - // before the snapshot which is not correct) - // - the implementation is simplified as we have a total control of what - // this pool is executing. Also, we can simply queue multiple tasks - // and be sure of the execution order. - // Not having mulitple possible threads in the same client allows us - // to ignore concurrency problems inside the client. - utils::ThreadPool thread_pool_{1}; - std::atomic replica_state_{replication::ReplicaState::INVALID}; - - utils::Scheduler replica_checker_; - Storage *storage_; - - memgraph::replication::ReplicationEpoch const *repl_epoch_; + ::memgraph::replication::ReplicationClient &client_; + // TODO Do not store the stream, make is a local variable + std::optional + replica_stream_; // Currently active stream (nullopt if not in use), note: a single stream per rpc client + mutable utils::Synchronized replica_state_{ + replication::ReplicaState::MAYBE_BEHIND}; }; } // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_storage_state.cpp b/src/storage/v2/replication/replication_storage_state.cpp index 1cd0bec09..a443c7171 100644 --- a/src/storage/v2/replication/replication_storage_state.cpp +++ b/src/storage/v2/replication/replication_storage_state.cpp @@ -16,10 +16,10 @@ namespace memgraph::storage { -void ReplicationStorageState::InitializeTransaction(uint64_t seq_num) { - replication_clients_.WithLock([&](auto &clients) { +void ReplicationStorageState::InitializeTransaction(uint64_t seq_num, Storage *storage) { + replication_clients_.WithLock([=](auto &clients) { for (auto &client : clients) { - client->StartTransactionReplication(seq_num); + client->StartTransactionReplication(seq_num, storage); } }); } @@ -52,12 +52,12 @@ void ReplicationStorageState::AppendOperation(durability::StorageMetadataOperati }); } -bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp) { +bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp, Storage *storage) { return replication_clients_.WithLock([=](auto &clients) { bool finalized_on_all_replicas = true; for (ReplicationClientPtr &client : clients) { client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(timestamp); }); - const auto finalized = client->FinalizeTransactionReplication(); + const auto finalized = client->FinalizeTransactionReplication(storage); if (client->Mode() == memgraph::replication::ReplicationMode::SYNC) { finalized_on_all_replicas = finalized && finalized_on_all_replicas; @@ -78,12 +78,12 @@ std::optional ReplicationStorageState::GetReplicaStat }); } -std::vector ReplicationStorageState::ReplicasInfo() const { - return replication_clients_.WithReadLock([](auto const &clients) { +std::vector ReplicationStorageState::ReplicasInfo(const Storage *storage) const { + return replication_clients_.WithReadLock([storage](auto const &clients) { std::vector replica_infos; replica_infos.reserve(clients.size()); - auto const asReplicaInfo = [](ReplicationClientPtr const &client) -> ReplicaInfo { - return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo()}; + auto const asReplicaInfo = [storage](ReplicationClientPtr const &client) -> ReplicaInfo { + return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo(storage)}; }; std::transform(clients.begin(), clients.end(), std::back_inserter(replica_infos), asReplicaInfo); return replica_infos; diff --git a/src/storage/v2/replication/replication_storage_state.hpp b/src/storage/v2/replication/replication_storage_state.hpp index afedb3950..e3d6b94a0 100644 --- a/src/storage/v2/replication/replication_storage_state.hpp +++ b/src/storage/v2/replication/replication_storage_state.hpp @@ -12,11 +12,13 @@ #pragma once #include +#include #include "kvstore/kvstore.hpp" #include "storage/v2/delta.hpp" #include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/transaction.hpp" +#include "utils/exceptions.hpp" #include "utils/result.hpp" /// REPLICATION /// @@ -33,21 +35,21 @@ namespace memgraph::storage { class Storage; -class ReplicationClient; +class ReplicationStorageClient; struct ReplicationStorageState { // Only MAIN can send - void InitializeTransaction(uint64_t seq_num); + void InitializeTransaction(uint64_t seq_num, Storage *storage); void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp); void AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp); void AppendOperation(durability::StorageMetadataOperation operation, LabelId label, const std::set &properties, const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t final_commit_timestamp); - bool FinalizeTransaction(uint64_t timestamp); + bool FinalizeTransaction(uint64_t timestamp, Storage *storage); // Getters auto GetReplicaState(std::string_view name) const -> std::optional; - auto ReplicasInfo() const -> std::vector; + auto ReplicasInfo(const Storage *storage) const -> std::vector; // History void TrackLatestHistory(); @@ -55,6 +57,19 @@ struct ReplicationStorageState { void Reset(); + template + bool WithClient(std::string_view replica_name, F &&callback) { + return replication_clients_.WithLock([replica_name, cb = std::forward(callback)](auto &clients) { + for (const auto &client : clients) { + if (client->Name() == replica_name) { + cb(client.get()); + return true; + } + } + return false; + }); + } + // Questions: // - storage durability <- databases/*name*/wal and snapshots (where this for epoch_id) // - multi-tenant durability <- databases/.durability (there is a list of all active tenants) @@ -74,7 +89,7 @@ struct ReplicationStorageState { // This way we can initialize client in main thread which means // that we can immediately notify the user if the initialization // failed. - using ReplicationClientPtr = std::unique_ptr; + using ReplicationClientPtr = std::unique_ptr; using ReplicationClientList = utils::Synchronized, utils::RWSpinLock>; ReplicationClientList replication_clients_; diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 8d8b06cd6..4142da3ca 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -109,7 +109,7 @@ struct EdgeInfoForDeletion { class Storage { friend class ReplicationServer; - friend class ReplicationClient; + friend class ReplicationStorageClient; public: Storage(Config config, StorageMode storage_mode); @@ -355,11 +355,7 @@ class Storage { virtual void PrepareForNewEpoch() = 0; - virtual auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr = 0; - - auto ReplicasInfo() const { return repl_storage_state_.ReplicasInfo(); } + auto ReplicasInfo() const { return repl_storage_state_.ReplicasInfo(this); } auto GetReplicaState(std::string_view name) const -> std::optional { return repl_storage_state_.GetReplicaState(name); } @@ -384,7 +380,7 @@ class Storage { Config config_; // Transaction engine - utils::SpinLock engine_lock_; + mutable utils::SpinLock engine_lock_; uint64_t timestamp_{kTimestampInitialId}; uint64_t transaction_id_{kTransactionInitialId}; diff --git a/tests/e2e/replication/show_while_creating_invalid_state.py b/tests/e2e/replication/show_while_creating_invalid_state.py index 996955dc1..74dcbce74 100644 --- a/tests/e2e/replication/show_while_creating_invalid_state.py +++ b/tests/e2e/replication/show_while_creating_invalid_state.py @@ -123,6 +123,143 @@ def test_show_replicas(connection): assert actual_data == expected_data +def test_drop_replicas(connection): + # Goal of this test is to check the DROP REPLICAS command. + # 0/ Manually start main and all replicas + # 1/ Check status of the replicas + # 2/ Kill replica 3 + # 3/ Drop replica 3 and check status + # 4/ Stop replica 4 + # 5/ Drop replica 4 and check status + # 6/ Kill replica 1 + # 7/ Drop replica 1 and check status + # 8/ Stop replica 2 + # 9/ Drop replica 2 and check status + # 10/ Restart all replicas + # 11/ Register them + # 12/ Drop all and check status + + def retrieve_data(): + return set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + # 0/ + interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION) + + cursor = connection(7687, "main").cursor() + + # 1/ + actual_data = set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + EXPECTED_COLUMN_NAMES = { + "name", + "socket_address", + "sync_mode", + "current_timestamp_of_replica", + "number_of_timestamp_behind_master", + "state", + } + + actual_column_names = {x.name for x in cursor.description} + assert actual_column_names == EXPECTED_COLUMN_NAMES + + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 2/ + interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_3") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "invalid"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 3/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_3") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 4/ + interactive_mg_runner.stop(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_4") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "invalid"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 5/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_4") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 6/ + interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_1") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "invalid"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 7/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_1") + expected_data = { + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 8/ + interactive_mg_runner.stop(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_2") + expected_data = { + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "invalid"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 9/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_2") + expected_data = set() + mg_sleep_and_assert(expected_data, retrieve_data) + + # 10/ + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_1") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_2") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_3") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_4") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_3 ASYNC TO '127.0.0.1:10003';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_4 ASYNC TO '127.0.0.1:10004';") + + # 11/ + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 12/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_1") + execute_and_fetch_all(cursor, "DROP REPLICA replica_2") + execute_and_fetch_all(cursor, "DROP REPLICA replica_3") + execute_and_fetch_all(cursor, "DROP REPLICA replica_4") + expected_data = set() + mg_sleep_and_assert(expected_data, retrieve_data) + + def test_basic_recovery(connection): # Goal of this test is to check the recovery of main. # 0/ We start all replicas manually: we want to be able to kill them ourselves without relying on external tooling to kill processes. @@ -630,10 +767,26 @@ def test_async_replication_when_main_is_killed(): ) # 2/ - for index in range(50): + # First make sure that anything has been replicated + for index in range(0, 5): + interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(f"CREATE (p:Number {{name:{index}}})") + expected_data = [("async_replica", "127.0.0.1:10001", "async", "ready")] + + def retrieve_data(): + replicas = interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query("SHOW REPLICAS;") + return [ + (replica_name, ip, mode, status) + for replica_name, ip, mode, timestamp, timestamp_behind_main, status in replicas + ] + + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + for index in range(5, 50): interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(f"CREATE (p:Number {{name:{index}}})") if random.randint(0, 100) > 95: main_killed = f"Main was killed at index={index}" + print(main_killed) interactive_mg_runner.kill(CONFIGURATION, "main") break diff --git a/tests/integration/telemetry/client.cpp b/tests/integration/telemetry/client.cpp index 558e0a6bc..34e1c2a67 100644 --- a/tests/integration/telemetry/client.cpp +++ b/tests/integration/telemetry/client.cpp @@ -41,7 +41,7 @@ int main(int argc, char **argv) { memgraph::storage::UpdatePaths(db_config, data_directory); memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(db_config)); - memgraph::dbms::DbmsHandler dbms_handler(db_config, repl_state + memgraph::dbms::DbmsHandler dbms_handler(db_config #ifdef MG_ENTERPRISE , &auth_, false, false diff --git a/tests/unit/dbms_handler.cpp b/tests/unit/dbms_handler.cpp index 75efddefe..a811e4159 100644 --- a/tests/unit/dbms_handler.cpp +++ b/tests/unit/dbms_handler.cpp @@ -52,18 +52,15 @@ class TestEnvironment : public ::testing::Environment { auth = std::make_unique>( storage_directory / "auth"); - repl_state_.emplace(memgraph::storage::ReplicationStateRootPath(storage_conf)); - ptr_ = std::make_unique(storage_conf, *repl_state_, auth.get(), false, true); + ptr_ = std::make_unique(storage_conf, auth.get(), false, true); } void TearDown() override { ptr_.reset(); auth.reset(); - repl_state_.reset(); } static std::unique_ptr ptr_; - std::optional repl_state_; }; std::unique_ptr TestEnvironment::ptr_ = nullptr; diff --git a/tests/unit/dbms_handler_community.cpp b/tests/unit/dbms_handler_community.cpp index efce2854d..3848cd347 100644 --- a/tests/unit/dbms_handler_community.cpp +++ b/tests/unit/dbms_handler_community.cpp @@ -52,18 +52,15 @@ class TestEnvironment : public ::testing::Environment { auth = std::make_unique>( storage_directory / "auth"); - repl_state_.emplace(memgraph::storage::ReplicationStateRootPath(storage_conf)); - ptr_ = std::make_unique(storage_conf, *repl_state_); + ptr_ = std::make_unique(storage_conf); } void TearDown() override { ptr_.reset(); auth.reset(); - repl_state_.reset(); } static std::unique_ptr ptr_; - std::optional repl_state_; }; std::unique_ptr TestEnvironment::ptr_ = nullptr; diff --git a/tests/unit/storage_v2_replication.cpp b/tests/unit/storage_v2_replication.cpp index 261b2ccf0..f07130c4a 100644 --- a/tests/unit/storage_v2_replication.cpp +++ b/tests/unit/storage_v2_replication.cpp @@ -102,8 +102,7 @@ class ReplicationTest : public ::testing::Test { struct MinMemgraph { MinMemgraph(const memgraph::storage::Config &conf) - : repl_state{ReplicationStateRootPath(conf)}, - dbms{conf, repl_state + : dbms{conf #ifdef MG_ENTERPRISE , reinterpret_cast< @@ -111,11 +110,12 @@ struct MinMemgraph { true, false #endif }, + repl_state{dbms.ReplicationState()}, db{*dbms.Get().get()}, - repl_handler(repl_state, dbms) { + repl_handler(dbms) { } - memgraph::replication::ReplicationState repl_state; memgraph::dbms::DbmsHandler dbms; + memgraph::replication::ReplicationState &repl_state; memgraph::dbms::Database &db; ReplicationHandler repl_handler; }; @@ -130,14 +130,13 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) { .port = ports[0], }); - ASSERT_FALSE(main.repl_handler - .RegisterReplica(ReplicationClientConfig{ - .name = "REPLICA", - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) - .HasError()); + const auto ® = main.repl_handler.RegisterReplica(ReplicationClientConfig{ + .name = "REPLICA", + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }); + ASSERT_FALSE(reg.HasError()) << (int)reg.GetError(); // vertex create // vertex add label @@ -966,14 +965,14 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAfterDroppingReplica) { .ip_address = local_host, .port = ports[0], }); - ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(res.HasError()) << (int)res.GetError(); res = main->repl_handler.RegisterReplica(ReplicationClientConfig{ .name = replicas[1], .mode = ReplicationMode::SYNC, .ip_address = local_host, .port = ports[1], }); - ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(res.HasError()) << (int)res.GetError(); auto replica_infos = main->db.storage()->ReplicasInfo(); From 70db2fca565463fe4137f98169aaf37e562d1a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Thu, 23 Nov 2023 13:46:04 +0100 Subject: [PATCH 26/52] Change package_all to package_memgraph (#1507) Add the ability to pick a specific package to build --- .github/workflows/package_all.yaml | 263 ------------------ ...ge_specific.yaml => package_memgraph.yaml} | 33 +-- 2 files changed, 17 insertions(+), 279 deletions(-) delete mode 100644 .github/workflows/package_all.yaml rename .github/workflows/{package_specific.yaml => package_memgraph.yaml} (87%) diff --git a/.github/workflows/package_all.yaml b/.github/workflows/package_all.yaml deleted file mode 100644 index f1831616a..000000000 --- a/.github/workflows/package_all.yaml +++ /dev/null @@ -1,263 +0,0 @@ -name: Package All - -# TODO(gitbuda): Cleanup docker container if GHA job was canceled. - -on: - workflow_dispatch: - inputs: - memgraph_version: - description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'" - required: false - build_type: - type: choice - description: "Memgraph Build type. Default value is Release." - default: 'Release' - options: - - Release - - RelWithDebInfo - -jobs: - centos-7: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: centos-7 - path: build/output/centos-7/memgraph*.rpm - - centos-9: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: centos-9 - path: build/output/centos-9/memgraph*.rpm - - debian-10: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-10 - path: build/output/debian-10/memgraph*.deb - - debian-11: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11 - path: build/output/debian-11/memgraph*.deb - - docker: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - cd release/package - ./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker - ./run.sh docker - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: docker - path: build/output/docker/memgraph*.tar.gz - - ubuntu-1804: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-18.04 - path: build/output/ubuntu-18.04/memgraph*.deb - - ubuntu-2004: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-20.04 - path: build/output/ubuntu-20.04/memgraph*.deb - - ubuntu-2204: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-22.04 - path: build/output/ubuntu-22.04/memgraph*.deb - - debian-11-platform: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11-platform - path: build/output/debian-11/memgraph*.deb - - fedora-36: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: fedora-36 - path: build/output/fedora-36/memgraph*.rpm - - amzn-2: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: amzn-2 - path: build/output/amzn-2/memgraph*.rpm - - debian-11-arm: - runs-on: [self-hosted, DockerMgBuild, ARM64, strange] - timeout-minutes: 120 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11-aarch64 - path: build/output/debian-11-arm/memgraph*.deb - - ubuntu-2204-arm: - runs-on: [self-hosted, DockerMgBuild, ARM64, strange] - timeout-minutes: 120 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-22.04-aarch64 - path: build/output/ubuntu-22.04-arm/memgraph*.deb - - upload-to-s3: - # only run upload if we specified version. Allows for runs without upload - if: "${{ github.event.inputs.memgraph_version != '' }}" - needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm] - runs-on: ubuntu-latest - steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - # name: # if name input parameter is not provided, all artifacts are downloaded - # and put in directories named after each one. - path: build/output/release - - name: Upload to S3 - uses: jakejarvis/s3-sync-action@v0.5.1 - env: - AWS_S3_BUCKET: "download.memgraph.com" - AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "eu-west-1" - SOURCE_DIR: "build/output/release" - DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/" diff --git a/.github/workflows/package_specific.yaml b/.github/workflows/package_memgraph.yaml similarity index 87% rename from .github/workflows/package_specific.yaml rename to .github/workflows/package_memgraph.yaml index c599f65ef..48a61ca53 100644 --- a/.github/workflows/package_specific.yaml +++ b/.github/workflows/package_memgraph.yaml @@ -1,4 +1,4 @@ -name: Package Specific +name: Package memgraph # TODO(gitbuda): Cleanup docker container if GHA job was canceled. @@ -10,16 +10,17 @@ on: required: false build_type: type: choice - description: "Memgraph Build type. Default value is Release." + description: "Memgraph Build type. Default value is Release" default: 'Release' options: - Release - RelWithDebInfo target_os: type: choice - description: "Target OS for which memgraph will be packaged. Default is Ubuntu 22.04" + description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04" default: 'ubuntu-22_04' options: + - all - amzn-2 - centos-7 - centos-9 @@ -36,7 +37,7 @@ on: jobs: amzn-2: - if: ${{ github.event.inputs.target_os == 'amzn-2' }} + if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -54,7 +55,7 @@ jobs: path: build/output/amzn-2/memgraph*.rpm centos-7: - if: ${{ github.event.inputs.target_os == 'centos-7' }} + if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -72,7 +73,7 @@ jobs: path: build/output/centos-7/memgraph*.rpm centos-9: - if: ${{ github.event.inputs.target_os == 'centos-9' }} + if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -90,7 +91,7 @@ jobs: path: build/output/centos-9/memgraph*.rpm debian-10: - if: ${{ github.event.inputs.target_os == 'debian-10' }} + if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -108,7 +109,7 @@ jobs: path: build/output/debian-10/memgraph*.deb debian-11: - if: ${{ github.event.inputs.target_os == 'debian-11' }} + if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -126,7 +127,7 @@ jobs: path: build/output/debian-11/memgraph*.deb debian-11-arm: - if: ${{ github.event.inputs.target_os == 'debian-11-arm' }} + if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: @@ -144,7 +145,7 @@ jobs: path: build/output/debian-11-arm/memgraph*.deb debian-11-platform: - if: ${{ github.event.inputs.target_os == 'debian-11-platform' }} + if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -162,7 +163,7 @@ jobs: path: build/output/debian-11/memgraph*.deb docker: - if: ${{ github.event.inputs.target_os == 'docker' }} + if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -182,7 +183,7 @@ jobs: path: build/output/docker/memgraph*.tar.gz fedora-36: - if: ${{ github.event.inputs.target_os == 'fedora-36' }} + if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -200,7 +201,7 @@ jobs: path: build/output/fedora-36/memgraph*.rpm ubuntu-18_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -218,7 +219,7 @@ jobs: path: build/output/ubuntu-18.04/memgraph*.deb ubuntu-20_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -236,7 +237,7 @@ jobs: path: build/output/ubuntu-20.04/memgraph*.deb ubuntu-22_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -254,7 +255,7 @@ jobs: path: build/output/ubuntu-22.04/memgraph*.deb ubuntu-22_04-arm: - if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: From 7f5a55f1b2c724648bf52b799efd3f6f47152560 Mon Sep 17 00:00:00 2001 From: Andi Date: Fri, 24 Nov 2023 13:11:47 +0100 Subject: [PATCH 27/52] Fix restarts when using init-file flag (#1465) --- src/memgraph.cpp | 11 ++-- src/query/exceptions.hpp | 6 +++ src/query/interpreter.cpp | 2 +- tests/integration/CMakeLists.txt | 26 +--------- tests/integration/init_file/CMakeLists.txt | 6 +++ tests/integration/init_file/auth.cypherl | 1 + tests/integration/init_file/runner.py | 60 ++++++++++++++++++++++ tests/integration/init_file/tester.cpp | 47 +++++++++++++++++ 8 files changed, 129 insertions(+), 30 deletions(-) create mode 100644 tests/integration/init_file/CMakeLists.txt create mode 100644 tests/integration/init_file/auth.cypherl create mode 100644 tests/integration/init_file/runner.py create mode 100644 tests/integration/init_file/tester.cpp diff --git a/src/memgraph.cpp b/src/memgraph.cpp index ce43f634d..bd0d3a2e1 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -65,10 +65,13 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbm std::string line; while (std::getline(file, line)) { if (!line.empty()) { - auto results = interpreter.Prepare(line, {}, {}); - memgraph::query::DiscardValueResultStream stream; - interpreter.Pull(&stream, {}, results.qid); - + try { + auto results = interpreter.Prepare(line, {}, {}); + memgraph::query::DiscardValueResultStream stream; + interpreter.Pull(&stream, {}, results.qid); + } catch (const memgraph::query::UserAlreadyExistsException &e) { + spdlog::warn("{} The rest of the init-file will be run.", e.what()); + } if (audit_log) { audit_log->Record("", "", line, {}, memgraph::dbms::kDefaultDB); } diff --git a/src/query/exceptions.hpp b/src/query/exceptions.hpp index 1b2e712f9..ac8cc8fe8 100644 --- a/src/query/exceptions.hpp +++ b/src/query/exceptions.hpp @@ -126,6 +126,12 @@ class InfoInMulticommandTxException : public QueryException { SPECIALIZE_GET_EXCEPTION_NAME(InfoInMulticommandTxException) }; +class UserAlreadyExistsException : public QueryException { + public: + using QueryException::QueryException; + SPECIALIZE_GET_EXCEPTION_NAME(UserAlreadyExistsException) +}; + /** * An exception for an illegal operation that can not be detected * before the query starts executing over data. diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index d957d4c2e..ad7535006 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -478,7 +478,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_ MG_ASSERT(password.IsString() || password.IsNull()); if (!auth->CreateUser(username, password.IsString() ? std::make_optional(std::string(password.ValueString())) : std::nullopt)) { - throw QueryRuntimeException("User '{}' already exists.", username); + throw UserAlreadyExistsException("User '{}' already exists.", username); } // If the license is not valid we create users with admin access diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 73d98ce6a..c61f046dc 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -1,38 +1,14 @@ -# telemetry test binaries add_subdirectory(telemetry) - -# ssl test binaries add_subdirectory(ssl) - -# transactions test binaries add_subdirectory(transactions) - -# auth test binaries add_subdirectory(auth) - -# lba test binaries add_subdirectory(fine_grained_access) - -# audit test binaries add_subdirectory(audit) - -# ldap test binaries add_subdirectory(ldap) - -# mg_import_csv test binaries add_subdirectory(mg_import_csv) - -# license_check test binaries add_subdirectory(license_info) - -#environment variable check binaries add_subdirectory(env_variable_check) - -#flag check binaries add_subdirectory(flag_check) - -#storage mode binaries add_subdirectory(storage_mode) - -#run time settings binaries add_subdirectory(run_time_settings) +add_subdirectory(init_file) diff --git a/tests/integration/init_file/CMakeLists.txt b/tests/integration/init_file/CMakeLists.txt new file mode 100644 index 000000000..41f2af6cc --- /dev/null +++ b/tests/integration/init_file/CMakeLists.txt @@ -0,0 +1,6 @@ +set(target_name memgraph__integration__init_file) +set(tester_target_name ${target_name}__tester) + +add_executable(${tester_target_name} tester.cpp) +set_target_properties(${tester_target_name} PROPERTIES OUTPUT_NAME tester) +target_link_libraries(${tester_target_name} mg-communication) diff --git a/tests/integration/init_file/auth.cypherl b/tests/integration/init_file/auth.cypherl new file mode 100644 index 000000000..3a2f8d441 --- /dev/null +++ b/tests/integration/init_file/auth.cypherl @@ -0,0 +1 @@ +CREATE USER memgraph1 IDENTIFIED BY '1234'; diff --git a/tests/integration/init_file/runner.py b/tests/integration/init_file/runner.py new file mode 100644 index 000000000..fcaa10f95 --- /dev/null +++ b/tests/integration/init_file/runner.py @@ -0,0 +1,60 @@ +import argparse +import os +import subprocess +import sys +import tempfile +import time + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +BUILD_DIR = os.path.join(PROJECT_DIR, "build") +INIT_FILE = os.path.join(SCRIPT_DIR, "auth.cypherl") +SIGNAL_SIGTERM = 15 + + +def wait_for_server(port, delay=0.1): + cmd = ["nc", "-z", "-w", "1", "127.0.0.1", str(port)] + while subprocess.call(cmd) != 0: + time.sleep(0.01) + time.sleep(delay) + + +def prepare_memgraph(memgraph_args): + memgraph = subprocess.Popen(list(map(str, memgraph_args))) + time.sleep(0.1) + assert memgraph.poll() is None, "Memgraph process died prematurely!" + wait_for_server(7687) + return memgraph + + +def terminate_memgraph(memgraph): + pid = memgraph.pid + try: + os.kill(pid, SIGNAL_SIGTERM) + except os.OSError: + assert False, "Memgraph process didn't exit cleanly!" + time.sleep(1) + + +def execute_test_restart_memgraph_with_init_file(memgraph_binary: str, tester_binary: str) -> None: + storage_directory = tempfile.TemporaryDirectory() + tester_args = [tester_binary, "--username", "memgraph1", "--password", "1234"] + memgraph = prepare_memgraph([memgraph_binary, "--data-directory", storage_directory.name, "--init-file", INIT_FILE]) + subprocess.run(tester_args, stdout=subprocess.PIPE, check=True).check_returncode() + terminate_memgraph(memgraph) + memgraph = prepare_memgraph([memgraph_binary, "--data-directory", storage_directory.name, "--init-file", INIT_FILE]) + subprocess.run(tester_args, stdout=subprocess.PIPE, check=True).check_returncode() + terminate_memgraph(memgraph) + + +if __name__ == "__main__": + memgraph_binary = os.path.join(PROJECT_DIR, "build", "memgraph") + tester_binary = os.path.join(BUILD_DIR, "tests", "integration", "init_file", "tester") + + parser = argparse.ArgumentParser() + parser.add_argument("--memgraph", default=memgraph_binary) + parser.add_argument("--tester", default=tester_binary) + args = parser.parse_args() + + execute_test_restart_memgraph_with_init_file(args.memgraph, args.tester) + sys.exit(0) diff --git a/tests/integration/init_file/tester.cpp b/tests/integration/init_file/tester.cpp new file mode 100644 index 000000000..d4486ead5 --- /dev/null +++ b/tests/integration/init_file/tester.cpp @@ -0,0 +1,47 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +#include "communication/bolt/client.hpp" +#include "io/network/endpoint.hpp" +#include "io/network/utils.hpp" +#include "utils/logging.hpp" + +DEFINE_string(address, "127.0.0.1", "Server address"); +DEFINE_int32(port, 7687, "Server port"); +DEFINE_string(username, "", "Username for the database"); +DEFINE_string(password, "", "Password for the database"); +DEFINE_bool(use_ssl, false, "Set to true to connect with SSL to the server."); + +// NOLINTNEXTLINE(bugprone-exception-escape) +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + memgraph::communication::SSLInit sslInit; + + memgraph::io::network::Endpoint endpoint(memgraph::io::network::ResolveHostname(FLAGS_address), FLAGS_port); + + memgraph::communication::ClientContext context(FLAGS_use_ssl); + memgraph::communication::bolt::Client client(context); + + client.Connect(endpoint, FLAGS_username, FLAGS_password); + auto ret = client.Execute("SHOW USERS", {}); + auto size = ret.records.size(); + MG_ASSERT(size == 1, "Too much users returned for SHOW USERA (got {}, expected 1)!", size); + auto row0_size = ret.records[0].size(); + MG_ASSERT(row0_size == 1, "Too much entries in query dump row (got {}, expected 1)!", row0_size); + auto user = ret.records[0][0].ValueString(); + MG_ASSERT(user == "memgraph1", "Unexpected user returned for SHOW USERS (got {}, expected memgraph)!", user); + + return 0; +} From 72d47fc3bf1fded14812e597ca951a1d13b44971 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:44:12 +0100 Subject: [PATCH 28/52] Implement short circuiting of exists evaluation (#1539) --- src/glue/communication.cpp | 2 ++ src/query/common.cpp | 3 +- .../interpret/awesome_memgraph_functions.cpp | 1 + src/query/interpret/eval.hpp | 13 ++++++- src/query/plan/operator.cpp | 11 +++--- src/query/plan/rule_based_planner.cpp | 14 +++++++- src/query/procedure/mg_procedure_impl.cpp | 5 ++- src/query/typed_value.cpp | 23 +++++++++++- src/query/typed_value.hpp | 9 ++++- tests/unit/formatters.hpp | 2 ++ tests/unit/query_expression_evaluator.cpp | 36 +++++++++++++++++++ tests/unit/query_plan.cpp | 20 +++++++++++ tests/unit/query_plan_checker.hpp | 25 +++++++++++++ 13 files changed, 154 insertions(+), 10 deletions(-) diff --git a/src/glue/communication.cpp b/src/glue/communication.cpp index fdf5129f6..60181e877 100644 --- a/src/glue/communication.cpp +++ b/src/glue/communication.cpp @@ -127,6 +127,8 @@ storage::Result ToBoltValue(const query::TypedValue &value, const storage return Value(value.ValueLocalDateTime()); case query::TypedValue::Type::Duration: return Value(value.ValueDuration()); + case query::TypedValue::Type::Function: + throw communication::bolt::ValueException("Unsupported conversion from TypedValue::Function to Value"); case query::TypedValue::Type::Graph: auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view); if (maybe_graph.HasError()) return maybe_graph.GetError(); diff --git a/src/query/common.cpp b/src/query/common.cpp index 793ae8044..3c75ed5ec 100644 --- a/src/query/common.cpp +++ b/src/query/common.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -62,6 +62,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b) { case TypedValue::Type::Edge: case TypedValue::Type::Path: case TypedValue::Type::Graph: + case TypedValue::Type::Function: throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type()); case TypedValue::Type::Null: LOG_FATAL("Invalid type"); diff --git a/src/query/interpret/awesome_memgraph_functions.cpp b/src/query/interpret/awesome_memgraph_functions.cpp index 2cfd11f8c..6f49ee99f 100644 --- a/src/query/interpret/awesome_memgraph_functions.cpp +++ b/src/query/interpret/awesome_memgraph_functions.cpp @@ -593,6 +593,7 @@ TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContex case TypedValue::Type::Duration: return TypedValue("DURATION", ctx.memory); case TypedValue::Type::Graph: + case TypedValue::Type::Function: throw QueryRuntimeException("Cannot fetch graph as it is not standardized openCypher type name"); } } diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index 916082bb2..e09ddcc97 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -903,7 +904,17 @@ class ExpressionEvaluator : public ExpressionVisitor { return TypedValue(std::move(result), ctx_->memory); } - TypedValue Visit(Exists &exists) override { return TypedValue{frame_->at(symbol_table_->at(exists)), ctx_->memory}; } + TypedValue Visit(Exists &exists) override { + TypedValue &frame_exists_value = frame_->at(symbol_table_->at(exists)); + if (!frame_exists_value.IsFunction()) [[unlikely]] { + throw QueryRuntimeException( + "Unexpected behavior: Exists expected a function, got {}. Please report the problem on GitHub issues", + frame_exists_value.type()); + } + TypedValue result{ctx_->memory}; + frame_exists_value.ValueFunction()(&result); + return result; + } TypedValue Visit(All &all) override { auto list_value = all.list_expression_->Accept(*this); diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 1c8d021c7..63bf5cd40 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -2500,13 +2500,16 @@ std::vector EvaluatePatternFilter::ModifiedSymbols(const SymbolTable &ta } bool EvaluatePatternFilter::EvaluatePatternFilterCursor::Pull(Frame &frame, ExecutionContext &context) { - OOMExceptionEnabler oom_exception; SCOPED_PROFILE_OP("EvaluatePatternFilter"); + std::function function = [&frame, self = this->self_, input_cursor = this->input_cursor_.get(), + &context](TypedValue *return_value) { + OOMExceptionEnabler oom_exception; + input_cursor->Reset(); - input_cursor_->Reset(); - - frame[self_.output_symbol_] = TypedValue(input_cursor_->Pull(frame, context), context.evaluation_context.memory); + *return_value = TypedValue(input_cursor->Pull(frame, context), context.evaluation_context.memory); + }; + frame[self_.output_symbol_] = TypedValue(std::move(function)); return true; } diff --git a/src/query/plan/rule_based_planner.cpp b/src/query/plan/rule_based_planner.cpp index cd223dd8e..f3d0c1487 100644 --- a/src/query/plan/rule_based_planner.cpp +++ b/src/query/plan/rule_based_planner.cpp @@ -17,6 +17,7 @@ #include #include +#include "query/plan/preprocess.hpp" #include "utils/algorithm.hpp" #include "utils/exceptions.hpp" #include "utils/logging.hpp" @@ -516,14 +517,25 @@ bool HasBoundFilterSymbols(const std::unordered_set &bound_symbols, cons Expression *ExtractFilters(const std::unordered_set &bound_symbols, Filters &filters, AstStorage &storage) { Expression *filter_expr = nullptr; + std::vector and_joinable_filters{}; for (auto filters_it = filters.begin(); filters_it != filters.end();) { if (HasBoundFilterSymbols(bound_symbols, *filters_it)) { - filter_expr = impl::BoolJoin(storage, filter_expr, filters_it->expression); + and_joinable_filters.emplace_back(*filters_it); filters_it = filters.erase(filters_it); } else { filters_it++; } } + // Idea here is to join filters in a way + // that pattern filter ( exists() ) is at the end + // so if any of the AND filters before + // evaluate to false we don't need to + // evaluate pattern ( exists() ) filter + std::partition(and_joinable_filters.begin(), and_joinable_filters.end(), + [](const FilterInfo &filter_info) { return filter_info.type != FilterInfo::Type::Pattern; }); + for (auto &and_joinable_filter : and_joinable_filters) { + filter_expr = impl::BoolJoin(storage, filter_expr, and_joinable_filter.expression); + } return filter_expr; } diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index f87377ba5..ab2b3ae4b 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -313,6 +313,8 @@ mgp_value_type FromTypedValueType(memgraph::query::TypedValue::Type type) { return MGP_VALUE_TYPE_LOCAL_DATE_TIME; case memgraph::query::TypedValue::Type::Duration: return MGP_VALUE_TYPE_DURATION; + case memgraph::query::TypedValue::Type::Function: + throw std::logic_error{"mgp_value for TypedValue::Type::Function doesn't exist."}; case memgraph::query::TypedValue::Type::Graph: throw std::logic_error{"mgp_value for TypedValue::Type::Graph doesn't exist."}; } @@ -3672,7 +3674,8 @@ std::ostream &PrintValue(const TypedValue &value, std::ostream *stream) { case TypedValue::Type::Edge: case TypedValue::Type::Path: case TypedValue::Type::Graph: - LOG_FATAL("value must not be a graph element"); + case TypedValue::Type::Function: + LOG_FATAL("value must not be a graph|function element"); } } diff --git a/src/query/typed_value.cpp b/src/query/typed_value.cpp index 13db88e1c..91893d71c 100644 --- a/src/query/typed_value.cpp +++ b/src/query/typed_value.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -22,6 +22,7 @@ #include "storage/v2/temporal.hpp" #include "utils/exceptions.hpp" #include "utils/fnv.hpp" +#include "utils/logging.hpp" #include "utils/memory.hpp" namespace memgraph::query { @@ -215,6 +216,9 @@ TypedValue::TypedValue(const TypedValue &other, utils::MemoryResource *memory) : case Type::Duration: new (&duration_v) utils::Duration(other.duration_v); return; + case Type::Function: + new (&function_v) std::function(other.function_v); + return; case Type::Graph: auto *graph_ptr = utils::Allocator(memory_).new_object(*other.graph_v); new (&graph_v) std::unique_ptr(graph_ptr); @@ -268,6 +272,9 @@ TypedValue::TypedValue(TypedValue &&other, utils::MemoryResource *memory) : memo case Type::Duration: new (&duration_v) utils::Duration(other.duration_v); break; + case Type::Function: + new (&function_v) std::function(other.function_v); + break; case Type::Graph: if (other.GetMemoryResource() == memory_) { new (&graph_v) std::unique_ptr(std::move(other.graph_v)); @@ -343,6 +350,7 @@ DEFINE_VALUE_AND_TYPE_GETTERS(utils::Date, Date, date_v) DEFINE_VALUE_AND_TYPE_GETTERS(utils::LocalTime, LocalTime, local_time_v) DEFINE_VALUE_AND_TYPE_GETTERS(utils::LocalDateTime, LocalDateTime, local_date_time_v) DEFINE_VALUE_AND_TYPE_GETTERS(utils::Duration, Duration, duration_v) +DEFINE_VALUE_AND_TYPE_GETTERS(std::function, Function, function_v) Graph &TypedValue::ValueGraph() { if (type_ != Type::Graph) { @@ -417,6 +425,8 @@ std::ostream &operator<<(std::ostream &os, const TypedValue::Type &type) { return os << "duration"; case TypedValue::Type::Graph: return os << "graph"; + case TypedValue::Type::Function: + return os << "function"; } LOG_FATAL("Unsupported TypedValue::Type"); } @@ -569,6 +579,9 @@ TypedValue &TypedValue::operator=(const TypedValue &other) { case Type::Duration: new (&duration_v) utils::Duration(other.duration_v); return *this; + case Type::Function: + new (&function_v) std::function(other.function_v); + return *this; } LOG_FATAL("Unsupported TypedValue::Type"); } @@ -628,6 +641,9 @@ TypedValue &TypedValue::operator=(TypedValue &&other) noexcept(false) { case Type::Duration: new (&duration_v) utils::Duration(other.duration_v); break; + case Type::Function: + new (&function_v) std::function{other.function_v}; + break; case Type::Graph: if (other.GetMemoryResource() == memory_) { new (&graph_v) std::unique_ptr(std::move(other.graph_v)); @@ -676,6 +692,9 @@ void TypedValue::DestroyValue() { case Type::LocalDateTime: case Type::Duration: break; + case Type::Function: + std::destroy_at(&function_v); + break; case Type::Graph: { auto *graph = graph_v.release(); std::destroy_at(&graph_v); @@ -1153,6 +1172,8 @@ size_t TypedValue::Hash::operator()(const TypedValue &value) const { case TypedValue::Type::Duration: return utils::DurationHash{}(value.ValueDuration()); break; + case TypedValue::Type::Function: + throw TypedValueException("Unsupported hash function for Function"); case TypedValue::Type::Graph: throw TypedValueException("Unsupported hash function for Graph"); } diff --git a/src/query/typed_value.hpp b/src/query/typed_value.hpp index c215e2276..0af38cecc 100644 --- a/src/query/typed_value.hpp +++ b/src/query/typed_value.hpp @@ -84,7 +84,8 @@ class TypedValue { LocalTime, LocalDateTime, Duration, - Graph + Graph, + Function }; // TypedValue at this exact moment of compilation is an incomplete type, and @@ -420,6 +421,9 @@ class TypedValue { new (&graph_v) std::unique_ptr(graph_ptr); } + explicit TypedValue(std::function &&other) + : function_v(std::move(other)), type_(Type::Function) {} + /** * Construct with the value of other. * Default utils::NewDeleteResource() is used for allocations. After the move, @@ -451,6 +455,7 @@ class TypedValue { TypedValue &operator=(const utils::LocalTime &); TypedValue &operator=(const utils::LocalDateTime &); TypedValue &operator=(const utils::Duration &); + TypedValue &operator=(const std::function &); /** Copy assign other, utils::MemoryResource of `this` is used */ TypedValue &operator=(const TypedValue &other); @@ -506,6 +511,7 @@ class TypedValue { DECLARE_VALUE_AND_TYPE_GETTERS(utils::LocalDateTime, LocalDateTime) DECLARE_VALUE_AND_TYPE_GETTERS(utils::Duration, Duration) DECLARE_VALUE_AND_TYPE_GETTERS(Graph, Graph) + DECLARE_VALUE_AND_TYPE_GETTERS(std::function, Function) #undef DECLARE_VALUE_AND_TYPE_GETTERS @@ -550,6 +556,7 @@ class TypedValue { utils::Duration duration_v; // As the unique_ptr is not allocator aware, it requires special attention when copying or moving graphs std::unique_ptr graph_v; + std::function function_v; }; /** diff --git a/tests/unit/formatters.hpp b/tests/unit/formatters.hpp index a5ee49166..5217fd65c 100644 --- a/tests/unit/formatters.hpp +++ b/tests/unit/formatters.hpp @@ -138,6 +138,8 @@ inline std::string ToString(const memgraph::query::TypedValue &value, const TAcc break; case memgraph::query::TypedValue::Type::Graph: throw std::logic_error{"Not implemented"}; + case memgraph::query::TypedValue::Type::Function: + throw std::logic_error{"Not implemented"}; } return os.str(); } diff --git a/tests/unit/query_expression_evaluator.cpp b/tests/unit/query_expression_evaluator.cpp index 44d3ed301..c9786fe5e 100644 --- a/tests/unit/query_expression_evaluator.cpp +++ b/tests/unit/query_expression_evaluator.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -83,6 +84,14 @@ class ExpressionEvaluatorTest : public ::testing::Test { return id; } + Exists *CreateExistsWithValue(std::string name, TypedValue &&value) { + auto id = storage.template Create(); + auto symbol = symbol_table.CreateSymbol(name, true); + id->MapTo(symbol); + frame[symbol] = std::move(value); + return id; + } + template auto Eval(TExpression *expr) { ctx.properties = NamesToProperties(storage.properties_, &dba); @@ -149,6 +158,33 @@ TYPED_TEST(ExpressionEvaluatorTest, AndOperatorShortCircuit) { } } +TYPED_TEST(ExpressionEvaluatorTest, AndExistsOperatorShortCircuit) { + { + std::function my_func = [](TypedValue * /*return_value*/) { + throw QueryRuntimeException("This should not be evaluated"); + }; + TypedValue func_should_not_evaluate{std::move(my_func)}; + + auto *op = this->storage.template Create( + this->storage.template Create(false), + this->CreateExistsWithValue("anon1", std::move(func_should_not_evaluate))); + auto value = this->Eval(op); + EXPECT_EQ(value.ValueBool(), false); + } + { + std::function my_func = [memory = this->ctx.memory](TypedValue *return_value) { + *return_value = TypedValue(false, memory); + }; + TypedValue should_evaluate{std::move(my_func)}; + + auto *op = + this->storage.template Create(this->storage.template Create(true), + this->CreateExistsWithValue("anon1", std::move(should_evaluate))); + auto value = this->Eval(op); + EXPECT_EQ(value.ValueBool(), false); + } +} + TYPED_TEST(ExpressionEvaluatorTest, AndOperatorNull) { { // Null doesn't short circuit diff --git a/tests/unit/query_plan.cpp b/tests/unit/query_plan.cpp index 910ebdc54..bc4b2660c 100644 --- a/tests/unit/query_plan.cpp +++ b/tests/unit/query_plan.cpp @@ -853,6 +853,26 @@ TYPED_TEST(TestPlanner, MatchFilterPropIsNotNull) { } } +TYPED_TEST(TestPlanner, MatchFilterWhere) { + // Test MATCH (n)-[r]-(m) WHERE exists((n)-[]-()) and n!=n and 7!=8 RETURN n + auto *query = QUERY(SINGLE_QUERY( + MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), + WHERE(AND(EXISTS(PATTERN(NODE("n"), EDGE("edge2", memgraph::query::EdgeAtom::Direction::BOTH, {}, false), + NODE("node3", std::nullopt, false))), + AND(NEQ(IDENT("n"), IDENT("n")), NEQ(LITERAL(7), LITERAL(8))))), + RETURN("n"))); + + std::list pattern_filter{new ExpectScanAll(), new ExpectExpand(), new ExpectLimit(), + new ExpectEvaluatePatternFilter()}; + CheckPlan( + query, this->storage, + ExpectFilter(), // 7!=8 + ExpectScanAll(), + ExpectFilter(std::vector>{pattern_filter}), // filter pulls from expand + ExpectExpand(), ExpectProduce()); + DeleteListContent(&pattern_filter); +} + TYPED_TEST(TestPlanner, MultiMatchWhere) { // Test MATCH (n) -[r]- (m) MATCH (l) WHERE n.prop < 42 RETURN n FakeDbAccessor dba; diff --git a/tests/unit/query_plan_checker.hpp b/tests/unit/query_plan_checker.hpp index 6f2f23df7..92089eb82 100644 --- a/tests/unit/query_plan_checker.hpp +++ b/tests/unit/query_plan_checker.hpp @@ -14,11 +14,13 @@ #include #include +#include "query/frontend/ast/ast.hpp" #include "query/frontend/semantic/symbol_generator.hpp" #include "query/frontend/semantic/symbol_table.hpp" #include "query/plan/operator.hpp" #include "query/plan/planner.hpp" #include "query/plan/preprocess.hpp" +#include "utils/typeinfo.hpp" namespace memgraph::query::plan { @@ -197,6 +199,29 @@ class ExpectFilter : public OpChecker { filter.pattern_filters_[i]->Accept(check_updates); } + // ordering in AND Operator must be ..., exists, exists, exists. + auto *expr = filter.expression_; + std::vector filter_expressions; + while (auto *and_operator = utils::Downcast(expr)) { + auto *expr1 = and_operator->expression1_; + auto *expr2 = and_operator->expression2_; + filter_expressions.emplace_back(expr1); + expr = expr2; + } + if (expr) filter_expressions.emplace_back(expr); + + auto it = filter_expressions.begin(); + for (; it != filter_expressions.end(); it++) { + if ((*it)->GetTypeInfo().name == query::Exists::kType.name) { + break; + } + } + while (it != filter_expressions.end()) { + ASSERT_TRUE((*it)->GetTypeInfo().name == query::Exists::kType.name) + << "Filter expression is '" << (*it)->GetTypeInfo().name << "' expected '" << query::Exists::kType.name + << "'!"; + it++; + } } std::vector> pattern_filters_; From bb2a7b8f213f8438fda61b397f3dcb00e19d28b2 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:00:34 +0100 Subject: [PATCH 29/52] Fix frame change collector incomplete pmr type (#1491) --- src/query/frame_change.hpp | 84 +++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/src/query/frame_change.hpp b/src/query/frame_change.hpp index 32fe1f36e..7baf1fe41 100644 --- a/src/query/frame_change.hpp +++ b/src/query/frame_change.hpp @@ -8,41 +8,42 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include +#include #include "query/typed_value.hpp" +#include "utils/fnv.hpp" #include "utils/memory.hpp" #include "utils/pmr/unordered_map.hpp" #include "utils/pmr/vector.hpp" namespace memgraph::query { // Key is hash output, value is vector of unique elements -using CachedType = utils::pmr::unordered_map>; +using CachedType = utils::pmr::unordered_map>; struct CachedValue { + using allocator_type = utils::Allocator; + // Cached value, this can be probably templateized CachedType cache_; - explicit CachedValue(utils::MemoryResource *mem) : cache_(mem) {} + explicit CachedValue(utils::MemoryResource *mem) : cache_{mem} {}; + CachedValue(const CachedValue &other, utils::MemoryResource *mem) : cache_(other.cache_, mem) {} + CachedValue(CachedValue &&other, utils::MemoryResource *mem) : cache_(std::move(other.cache_), mem){}; - CachedValue(CachedType &&cache, memgraph::utils::MemoryResource *memory) : cache_(std::move(cache), memory) {} + CachedValue(CachedValue &&other) noexcept : CachedValue(std::move(other), other.GetMemoryResource()) {} - CachedValue(const CachedValue &other, memgraph::utils::MemoryResource *memory) : cache_(other.cache_, memory) {} + CachedValue(const CachedValue &other) + : CachedValue(other, std::allocator_traits::select_on_container_copy_construction( + other.GetMemoryResource()) + .GetMemoryResource()) {} - CachedValue(CachedValue &&other, memgraph::utils::MemoryResource *memory) : cache_(std::move(other.cache_), memory) {} - - CachedValue(CachedValue &&other) noexcept = delete; - - /// Copy construction without memgraph::utils::MemoryResource is not allowed. - CachedValue(const CachedValue &) = delete; + utils::MemoryResource *GetMemoryResource() const { return cache_.get_allocator().GetMemoryResource(); } CachedValue &operator=(const CachedValue &) = delete; CachedValue &operator=(CachedValue &&) = delete; ~CachedValue() = default; - memgraph::utils::MemoryResource *GetMemoryResource() const noexcept { - return cache_.get_allocator().GetMemoryResource(); - } - bool CacheValue(const TypedValue &maybe_list) { if (!maybe_list.IsList()) { return false; @@ -70,7 +71,7 @@ struct CachedValue { } private: - static bool IsValueInVec(const std::vector &vec_values, const TypedValue &value) { + static bool IsValueInVec(const utils::pmr::vector &vec_values, const TypedValue &value) { return std::any_of(vec_values.begin(), vec_values.end(), [&value](auto &vec_value) { const auto is_value_equal = vec_value == value; if (is_value_equal.IsNull()) return false; @@ -82,35 +83,70 @@ struct CachedValue { // Class tracks keys for which user can cache values which help with faster search or faster retrieval // in the future. Used for IN LIST operator. class FrameChangeCollector { + /** Allocator type so that STL containers are aware that we need one */ + using allocator_type = utils::Allocator; + public: - explicit FrameChangeCollector() : tracked_values_(&memory_resource_){}; + explicit FrameChangeCollector(utils::MemoryResource *mem = utils::NewDeleteResource()) : tracked_values_{mem} {} + + FrameChangeCollector(FrameChangeCollector &&other, utils::MemoryResource *mem) + : tracked_values_(std::move(other.tracked_values_), mem) {} + FrameChangeCollector(const FrameChangeCollector &other, utils::MemoryResource *mem) + : tracked_values_(other.tracked_values_, mem) {} + + FrameChangeCollector(const FrameChangeCollector &other) + : FrameChangeCollector(other, std::allocator_traits::select_on_container_copy_construction( + other.GetMemoryResource()) + .GetMemoryResource()){}; + + FrameChangeCollector(FrameChangeCollector &&other) noexcept + : FrameChangeCollector(std::move(other), other.GetMemoryResource()) {} + + /** Copy assign other, utils::MemoryResource of `this` is used */ + FrameChangeCollector &operator=(const FrameChangeCollector &) = default; + + /** Move assign other, utils::MemoryResource of `this` is used. */ + FrameChangeCollector &operator=(FrameChangeCollector &&) noexcept = default; + + utils::MemoryResource *GetMemoryResource() const { return tracked_values_.get_allocator().GetMemoryResource(); } CachedValue &AddTrackingKey(const std::string &key) { - const auto &[it, _] = tracked_values_.emplace(key, tracked_values_.get_allocator().GetMemoryResource()); + const auto &[it, _] = tracked_values_.emplace( + std::piecewise_construct, std::forward_as_tuple(utils::pmr::string(key, utils::NewDeleteResource())), + std::forward_as_tuple()); return it->second; } - bool IsKeyTracked(const std::string &key) const { return tracked_values_.contains(key); } + bool IsKeyTracked(const std::string &key) const { + return tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource())); + } bool IsKeyValueCached(const std::string &key) const { - return IsKeyTracked(key) && !tracked_values_.at(key).cache_.empty(); + return IsKeyTracked(key) && !tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource())).cache_.empty(); } bool ResetTrackingValue(const std::string &key) { - if (!tracked_values_.contains(key)) { + if (!tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource()))) { return false; } - tracked_values_.erase(key); + tracked_values_.erase(utils::pmr::string(key, utils::NewDeleteResource())); AddTrackingKey(key); return true; } - CachedValue &GetCachedValue(const std::string &key) { return tracked_values_.at(key); } + CachedValue &GetCachedValue(const std::string &key) { + return tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource())); + } bool IsTrackingValues() const { return !tracked_values_.empty(); } + ~FrameChangeCollector() = default; + private: - utils::MonotonicBufferResource memory_resource_{0}; - memgraph::utils::pmr::unordered_map tracked_values_; + struct PmrStringHash { + size_t operator()(const utils::pmr::string &key) const { return utils::Fnv(key); } + }; + + utils::pmr::unordered_map tracked_values_; }; } // namespace memgraph::query From ac0c4193b03bd716b0dbdd77af46efdc67f638ce Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 28 Nov 2023 11:46:32 +0100 Subject: [PATCH 30/52] Remove comment --- src/query/interpreter.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 37482d2cf..7292c4591 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -3082,8 +3082,6 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici break; } - - // NODE_LABELS } return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), From b74aee186e37ef8ab904b1a9d87b18061e7c7be9 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 28 Nov 2023 13:34:21 +0100 Subject: [PATCH 31/52] Add tests for the retrieval queries --- tests/e2e/CMakeLists.txt | 1 + tests/e2e/metadata_queries/CMakeLists.txt | 7 ++ tests/e2e/metadata_queries/common.py | 34 ++++++++ .../metadata_queries/show_edge_types_info.py | 80 +++++++++++++++++++ .../metadata_queries/show_node_labels_info.py | 67 ++++++++++++++++ tests/e2e/metadata_queries/workloads.yaml | 18 +++++ 6 files changed, 207 insertions(+) create mode 100644 tests/e2e/metadata_queries/CMakeLists.txt create mode 100644 tests/e2e/metadata_queries/common.py create mode 100644 tests/e2e/metadata_queries/show_edge_types_info.py create mode 100644 tests/e2e/metadata_queries/show_node_labels_info.py create mode 100644 tests/e2e/metadata_queries/workloads.yaml diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 28fe94559..c7071678e 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -48,6 +48,7 @@ add_subdirectory(temporal_types) add_subdirectory(write_procedures) add_subdirectory(configuration) add_subdirectory(magic_functions) +add_subdirectory(metadata_queries) add_subdirectory(module_file_manager) add_subdirectory(monitoring_server) add_subdirectory(lba_procedures) diff --git a/tests/e2e/metadata_queries/CMakeLists.txt b/tests/e2e/metadata_queries/CMakeLists.txt new file mode 100644 index 000000000..9a91900cd --- /dev/null +++ b/tests/e2e/metadata_queries/CMakeLists.txt @@ -0,0 +1,7 @@ +function(copy_metadata_queries_e2e_python_files FILE_NAME) + copy_e2e_python_files(metadata_queries ${FILE_NAME}) +endfunction() + +copy_metadata_queries_e2e_python_files(common.py) +copy_metadata_queries_e2e_python_files(show_node_labels_info.py) +copy_metadata_queries_e2e_python_files(show_edge_types_info.py) diff --git a/tests/e2e/metadata_queries/common.py b/tests/e2e/metadata_queries/common.py new file mode 100644 index 000000000..f36c5ee16 --- /dev/null +++ b/tests/e2e/metadata_queries/common.py @@ -0,0 +1,34 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import typing + +import mgclient +import pytest + + +@pytest.fixture(scope="module") +def cursor(**kwargs) -> mgclient.Connection: + connection = mgclient.connect(host="localhost", port=7687, **kwargs) + connection.autocommit = True + return connection.cursor() + + +def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = dict()) -> typing.List[tuple]: + cursor.execute(query, params) + return cursor.fetchall() + + +def are_results_equal(result1, result2): + if len(result1) != len(result2): + return False + + return sorted(result1) == sorted(result2) diff --git a/tests/e2e/metadata_queries/show_edge_types_info.py b/tests/e2e/metadata_queries/show_edge_types_info.py new file mode 100644 index 000000000..f20f0beab --- /dev/null +++ b/tests/e2e/metadata_queries/show_edge_types_info.py @@ -0,0 +1,80 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys + +import pytest +from common import are_results_equal, cursor, execute_and_fetch_all + + +# Helper functions +def create_nodes(cursor): + execute_and_fetch_all( + cursor, "CREATE (charlie:Person:Actor {name: 'Charlie Sheen'}), (oliver:Person:Director {name: 'Oliver Stone'})" + ) + + +def create_edges(cursor): + execute_and_fetch_all( + cursor, + "MATCH (charlie:Person {name: 'Charlie Sheen'}), (oliver:Person {name: 'Oliver Stone'}) CREATE (charlie)-[:ACTED_IN {role: 'Bud Fox'}]->(wallStreet:Movie {title: 'Wall Street'})<-[:DIRECTED]-(oliver)", + ) + + +def edge_types_info(cursor): + return execute_and_fetch_all(cursor, "SHOW EDGE_TYPES INFO") + + +def default_expected_result(cursor): + return [("DIRECTED",), ("ACTED_IN",)] + + +# Tests +def test_return_empty(cursor): + create_nodes(cursor) + + edge_types = edge_types_info(cursor) + expected = [] + assert are_results_equal(expected, edge_types) + + +def test_return_edge_types_simple(cursor): + create_nodes(cursor) + create_edges(cursor) + + edge_types = edge_types_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, edge_types) + + +def test_return_edge_types_repeating_identical_edges(cursor): + create_nodes(cursor) + + for _ in range(100): + create_edges(cursor) + + edge_types = edge_types_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, edge_types) + + +def test_return_edge_types_obtainable_after_edge_deletion(cursor): + create_nodes(cursor) + create_edges(cursor) + execute_and_fetch_all(cursor, "MATCH(n) DETACH DELETE n") + + edge_types = edge_types_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, edge_types) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/metadata_queries/show_node_labels_info.py b/tests/e2e/metadata_queries/show_node_labels_info.py new file mode 100644 index 000000000..e98ef3e92 --- /dev/null +++ b/tests/e2e/metadata_queries/show_node_labels_info.py @@ -0,0 +1,67 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys + +import pytest +from common import are_results_equal, cursor, execute_and_fetch_all + + +# Helper functions +def create_nodes(cursor): + execute_and_fetch_all( + cursor, "CREATE (charlie:Person:Actor {name: 'Charlie Sheen'}), (oliver:Person:Director {name: 'Oliver Stone'})" + ) + + +def node_labels_info(cursor): + return execute_and_fetch_all(cursor, "SHOW NODE_LABELS INFO") + + +def default_expected_result(cursor): + return [("Person",), ("Actor",), ("Director",)] + + +# Tests +def test_return_empty(cursor): + node_labels = node_labels_info(cursor) + expected = [] + assert are_results_equal(expected, node_labels) + + +def test_return_node_labels_simple(cursor): + create_nodes(cursor) + + node_labels = node_labels_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, node_labels) + + +def test_return_node_labels_repeating_identical_labels(cursor): + for _ in range(100): + create_nodes(cursor) + + node_labels = node_labels_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, node_labels) + + +def test_return_node_labels_obtainable_after_vertex_deletion(cursor): + create_nodes(cursor) + execute_and_fetch_all(cursor, "MATCH(n) DELETE n") + + node_labels = node_labels_info(cursor) + expected = default_expected_result(cursor) + assert are_results_equal(expected, node_labels) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/metadata_queries/workloads.yaml b/tests/e2e/metadata_queries/workloads.yaml new file mode 100644 index 000000000..d4b213ae5 --- /dev/null +++ b/tests/e2e/metadata_queries/workloads.yaml @@ -0,0 +1,18 @@ +metadata_queries: &metadata_queries + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr", "--storage-enable-schema-metadata=TRUE"] + log_file: "metadata-queries.log" + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Show edge types info" + binary: "tests/e2e/pytest_runner.sh" + args: ["metadata_queries/show_edge_types_info.py"] + <<: *metadata_queries + + - name: "Show node labels info" + binary: "tests/e2e/pytest_runner.sh" + args: ["metadata_queries/show_node_labels_info.py"] + <<: *metadata_queries From 14f92b4a0fcce743246da0b73ef8db53a6e91399 Mon Sep 17 00:00:00 2001 From: Gareth Andrew Lloyd Date: Fri, 1 Dec 2023 12:38:48 +0000 Subject: [PATCH 32/52] Bugfix: correct replication handler (#1540) Fixes root cause of a cascade of failures in replication code: - Replica handling of deleting an edge is now corrected. Now tolerant of multiple edges of the same relationship type. - Improved robustness: correct exception handling around failed stream of current WAL file. This now means a REPLICA failure will no longer prevent transactions on MAIN from performing WAL writes. - Slightly better diagnostic messages, not user friendly but helps get developer to correct root cause quicker. - Proactively remove vertex+edges during Abort rather than defer to GC to do that work, this included fixing constraints and indexes to be safe. Co-authored-by: Andreja Tonev --- src/dbms/inmemory/replication_handlers.cpp | 86 ++++++---- src/integrations/pulsar/consumer.hpp | 3 +- src/storage/v2/constraints/constraints.cpp | 4 + src/storage/v2/constraints/constraints.hpp | 4 + .../v2/disk/edge_import_mode_cache.cpp | 7 +- src/storage/v2/disk/storage.cpp | 45 ++++++ src/storage/v2/disk/storage.hpp | 3 + src/storage/v2/indices/indices.cpp | 19 +++ src/storage/v2/indices/indices.hpp | 17 ++ .../v2/indices/label_property_index.hpp | 5 + src/storage/v2/inmemory/label_index.cpp | 46 +++++- src/storage/v2/inmemory/label_index.hpp | 15 +- .../v2/inmemory/label_property_index.cpp | 63 +++++++- .../v2/inmemory/label_property_index.hpp | 30 +++- src/storage/v2/inmemory/storage.cpp | 147 ++++++++++++++++-- src/storage/v2/inmemory/storage.hpp | 5 + .../v2/inmemory/unique_constraints.cpp | 33 +++- .../v2/inmemory/unique_constraints.hpp | 4 + src/storage/v2/storage.hpp | 3 + .../test_query_modules/module_test.cpp | 10 +- tests/e2e/replication/CMakeLists.txt | 1 + tests/e2e/replication/edge_delete.py | 56 +++++++ tests/e2e/replication/workloads.yaml | 22 +++ 23 files changed, 558 insertions(+), 70 deletions(-) create mode 100755 tests/e2e/replication/edge_delete.py diff --git a/src/dbms/inmemory/replication_handlers.cpp b/src/dbms/inmemory/replication_handlers.cpp index ce1f6da20..a5f56ee3d 100644 --- a/src/dbms/inmemory/replication_handlers.cpp +++ b/src/dbms/inmemory/replication_handlers.cpp @@ -370,8 +370,9 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage constexpr bool kSharedAccess = false; std::optional> commit_timestamp_and_accessor; - auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp, - bool unique = kSharedAccess) { + auto const get_transaction = [storage, &commit_timestamp_and_accessor]( + uint64_t commit_timestamp, + bool unique = kSharedAccess) -> storage::InMemoryStorage::ReplicationAccessor * { if (!commit_timestamp_and_accessor) { std::unique_ptr acc = nullptr; if (unique) { @@ -415,9 +416,11 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint()); auto *transaction = get_transaction(timestamp); auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW); - if (!vertex) throw utils::BasicException("Invalid transaction!"); + if (!vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto ret = transaction->DeleteVertex(&*vertex); - if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError() || !ret.GetValue()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::VERTEX_ADD_LABEL: { @@ -425,9 +428,11 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage delta.vertex_add_remove_label.label); auto *transaction = get_transaction(timestamp); auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW); - if (!vertex) throw utils::BasicException("Invalid transaction!"); + if (!vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto ret = vertex->AddLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label)); - if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError() || !ret.GetValue()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::VERTEX_REMOVE_LABEL: { @@ -435,9 +440,11 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage delta.vertex_add_remove_label.label); auto *transaction = get_transaction(timestamp); auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW); - if (!vertex) throw utils::BasicException("Invalid transaction!"); + if (!vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto ret = vertex->RemoveLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label)); - if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError() || !ret.GetValue()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::VERTEX_SET_PROPERTY: { @@ -445,10 +452,12 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value); auto *transaction = get_transaction(timestamp); auto vertex = transaction->FindVertex(delta.vertex_edge_set_property.gid, View::NEW); - if (!vertex) throw utils::BasicException("Invalid transaction!"); + if (!vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto ret = vertex->SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property), delta.vertex_edge_set_property.value); - if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::EDGE_CREATE: { @@ -457,13 +466,16 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); auto *transaction = get_transaction(timestamp); auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW); - if (!from_vertex) throw utils::BasicException("Invalid transaction!"); + if (!from_vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW); - if (!to_vertex) throw utils::BasicException("Invalid transaction!"); + if (!to_vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto edge = transaction->CreateEdgeEx(&*from_vertex, &*to_vertex, transaction->NameToEdgeType(delta.edge_create_delete.edge_type), delta.edge_create_delete.gid); - if (edge.HasError()) throw utils::BasicException("Invalid transaction!"); + if (edge.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::EDGE_DELETE: { @@ -472,16 +484,17 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); auto *transaction = get_transaction(timestamp); auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW); - if (!from_vertex) throw utils::BasicException("Invalid transaction!"); + if (!from_vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW); - if (!to_vertex) throw utils::BasicException("Invalid transaction!"); - auto edges = from_vertex->OutEdges(View::NEW, {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)}, - &*to_vertex); - if (edges.HasError()) throw utils::BasicException("Invalid transaction!"); - if (edges->edges.size() != 1) throw utils::BasicException("Invalid transaction!"); - auto &edge = (*edges).edges[0]; - auto ret = transaction->DeleteEdge(&edge); - if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); + if (!to_vertex) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); + auto edgeType = transaction->NameToEdgeType(delta.edge_create_delete.edge_type); + auto edge = + transaction->FindEdge(delta.edge_create_delete.gid, View::NEW, edgeType, &*from_vertex, &*to_vertex); + if (!edge) throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); + if (auto ret = transaction->DeleteEdge(&*edge); ret.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::EDGE_SET_PROPERTY: { @@ -498,7 +511,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage // yields an accessor that is only valid for managing the edge's // properties. auto edge = edge_acc.find(delta.vertex_edge_set_property.gid); - if (edge == edge_acc.end()) throw utils::BasicException("Invalid transaction!"); + if (edge == edge_acc.end()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); // The edge visibility check must be done here manually because we // don't allow direct access to the edges through the public API. { @@ -530,7 +544,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage } } }); - if (!is_visible) throw utils::BasicException("Invalid transaction!"); + if (!is_visible) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); } EdgeRef edge_ref(&*edge); // Here we create an edge accessor that we will use to get the @@ -543,7 +558,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property), delta.vertex_edge_set_property.value); - if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } @@ -553,7 +569,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage throw utils::BasicException("Invalid commit data!"); auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first, false /* not main */); - if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); commit_timestamp_and_accessor = std::nullopt; break; } @@ -563,14 +580,14 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage // Need to send the timestamp auto *transaction = get_transaction(timestamp, kUniqueAccess); if (transaction->CreateIndex(storage->NameToLabel(delta.operation_label.label)).HasError()) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::LABEL_INDEX_DROP: { spdlog::trace(" Drop label index on :{}", delta.operation_label.label); auto *transaction = get_transaction(timestamp, kUniqueAccess); if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError()) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::LABEL_INDEX_STATS_SET: { @@ -601,7 +618,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage ->CreateIndex(storage->NameToLabel(delta.operation_label_property.label), storage->NameToProperty(delta.operation_label_property.property)) .HasError()) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: { @@ -612,7 +629,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage ->DropIndex(storage->NameToLabel(delta.operation_label_property.label), storage->NameToProperty(delta.operation_label_property.property)) .HasError()) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: { @@ -644,7 +661,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage auto ret = transaction->CreateExistenceConstraint(storage->NameToLabel(delta.operation_label_property.label), storage->NameToProperty(delta.operation_label_property.property)); - if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); + if (ret.HasError()) + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { @@ -655,7 +673,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage ->DropExistenceConstraint(storage->NameToLabel(delta.operation_label_property.label), storage->NameToProperty(delta.operation_label_property.property)) .HasError()) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: { @@ -670,7 +688,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage auto ret = transaction->CreateUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), properties); if (!ret.HasValue() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); break; } case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { @@ -685,7 +703,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage auto ret = transaction->DropUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), properties); if (ret != UniqueConstraints::DeletionStatus::SUCCESS) { - throw utils::BasicException("Invalid transaction!"); + throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__); } break; } diff --git a/src/integrations/pulsar/consumer.hpp b/src/integrations/pulsar/consumer.hpp index 1caa366ad..06ed3a550 100644 --- a/src/integrations/pulsar/consumer.hpp +++ b/src/integrations/pulsar/consumer.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -10,6 +10,7 @@ // licenses/APL.txt. #pragma once + #include #include #include diff --git a/src/storage/v2/constraints/constraints.cpp b/src/storage/v2/constraints/constraints.cpp index 42128511c..6a6554db4 100644 --- a/src/storage/v2/constraints/constraints.cpp +++ b/src/storage/v2/constraints/constraints.cpp @@ -29,4 +29,8 @@ Constraints::Constraints(const Config &config, StorageMode storage_mode) { }; }); } + +void Constraints::AbortEntries(std::span vertices, uint64_t exact_start_timestamp) const { + static_cast(unique_constraints_.get())->AbortEntries(vertices, exact_start_timestamp); +} } // namespace memgraph::storage diff --git a/src/storage/v2/constraints/constraints.hpp b/src/storage/v2/constraints/constraints.hpp index 8469a5470..1f5ef999e 100644 --- a/src/storage/v2/constraints/constraints.hpp +++ b/src/storage/v2/constraints/constraints.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "storage/v2/config.hpp" #include "storage/v2/constraints/existence_constraints.hpp" #include "storage/v2/constraints/unique_constraints.hpp" @@ -27,6 +29,8 @@ struct Constraints { Constraints &operator=(Constraints &&) = delete; ~Constraints() = default; + void AbortEntries(std::span vertices, uint64_t exact_start_timestamp) const; + std::unique_ptr existence_constraints_; std::unique_ptr unique_constraints_; }; diff --git a/src/storage/v2/disk/edge_import_mode_cache.cpp b/src/storage/v2/disk/edge_import_mode_cache.cpp index 2a29a1606..cd1ca0dc2 100644 --- a/src/storage/v2/disk/edge_import_mode_cache.cpp +++ b/src/storage/v2/disk/edge_import_mode_cache.cpp @@ -10,7 +10,9 @@ // licenses/APL.txt. #include "storage/v2/disk//edge_import_mode_cache.hpp" + #include + #include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/indices/indices.hpp" #include "storage/v2/inmemory/label_index.hpp" @@ -28,7 +30,7 @@ EdgeImportModeCache::EdgeImportModeCache(const Config &config) InMemoryLabelIndex::Iterable EdgeImportModeCache::Vertices(LabelId label, View view, Storage *storage, Transaction *transaction) const { auto *mem_label_index = static_cast(in_memory_indices_.label_index_.get()); - return mem_label_index->Vertices(label, view, storage, transaction); + return mem_label_index->Vertices(label, vertices_.access(), view, storage, transaction); } InMemoryLabelPropertyIndex::Iterable EdgeImportModeCache::Vertices( @@ -37,7 +39,8 @@ InMemoryLabelPropertyIndex::Iterable EdgeImportModeCache::Vertices( Transaction *transaction) const { auto *mem_label_property_index = static_cast(in_memory_indices_.label_property_index_.get()); - return mem_label_property_index->Vertices(label, property, lower_bound, upper_bound, view, storage, transaction); + return mem_label_property_index->Vertices(label, property, vertices_.access(), lower_bound, upper_bound, view, + storage, transaction); } bool EdgeImportModeCache::CreateIndex(LabelId label, PropertyId property, diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 09b28943c..073d1fbd1 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -71,6 +71,37 @@ namespace memgraph::storage { +namespace { + +auto FindEdges(const View view, EdgeTypeId edge_type, const VertexAccessor *from_vertex, VertexAccessor *to_vertex) + -> Result { + auto use_out_edges = [](Vertex const *from_vertex, Vertex const *to_vertex) { + // Obtain the locks by `gid` order to avoid lock cycles. + auto guard_from = std::unique_lock{from_vertex->lock, std::defer_lock}; + auto guard_to = std::unique_lock{to_vertex->lock, std::defer_lock}; + if (from_vertex->gid < to_vertex->gid) { + guard_from.lock(); + guard_to.lock(); + } else if (from_vertex->gid > to_vertex->gid) { + guard_to.lock(); + guard_from.lock(); + } else { + // The vertices are the same vertex, only lock one. + guard_from.lock(); + } + + // With the potentially cheaper side FindEdges + const auto out_n = from_vertex->out_edges.size(); + const auto in_n = to_vertex->in_edges.size(); + return out_n <= in_n; + }; + + return use_out_edges(from_vertex->vertex_, to_vertex->vertex_) ? from_vertex->OutEdges(view, {edge_type}, to_vertex) + : to_vertex->InEdges(view, {edge_type}, from_vertex); +} + +} // namespace + using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; namespace { @@ -949,6 +980,20 @@ Result DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_); } +std::optional DiskStorage::DiskAccessor::FindEdge(Gid gid, View view, EdgeTypeId edge_type, + VertexAccessor *from_vertex, + VertexAccessor *to_vertex) { + auto res = FindEdges(view, edge_type, from_vertex, to_vertex); + if (res.HasError()) return std::nullopt; // TODO: use a Result type + + auto const it = std::ranges::find_if( + res->edges, [gid](EdgeAccessor const &edge_accessor) { return edge_accessor.edge_.ptr->gid == gid; }); + + if (it == res->edges.end()) return std::nullopt; // TODO: use a Result type + + return *it; +} + Result DiskStorage::DiskAccessor::EdgeSetFrom(EdgeAccessor * /*edge*/, VertexAccessor * /*new_from*/) { MG_ASSERT(false, "EdgeSetFrom is currently only implemented for InMemory storage"); return Error::NONEXISTENT_OBJECT; diff --git a/src/storage/v2/disk/storage.hpp b/src/storage/v2/disk/storage.hpp index e93566c09..8640462de 100644 --- a/src/storage/v2/disk/storage.hpp +++ b/src/storage/v2/disk/storage.hpp @@ -121,6 +121,9 @@ class DiskStorage final : public Storage { Result CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type) override; + std::optional FindEdge(Gid gid, View view, EdgeTypeId edge_type, VertexAccessor *from_vertex, + VertexAccessor *to_vertex) override; + Result EdgeSetFrom(EdgeAccessor *edge, VertexAccessor *new_from) override; Result EdgeSetTo(EdgeAccessor *edge, VertexAccessor *new_to) override; diff --git a/src/storage/v2/indices/indices.cpp b/src/storage/v2/indices/indices.cpp index bf7295de2..e0b194ad4 100644 --- a/src/storage/v2/indices/indices.cpp +++ b/src/storage/v2/indices/indices.cpp @@ -17,6 +17,21 @@ namespace memgraph::storage { +void Indices::AbortEntries(LabelId labelId, std::span vertices, uint64_t exact_start_timestamp) const { + static_cast(label_index_.get())->AbortEntries(labelId, vertices, exact_start_timestamp); +} + +void Indices::AbortEntries(PropertyId property, std::span const> vertices, + uint64_t exact_start_timestamp) const { + static_cast(label_property_index_.get()) + ->AbortEntries(property, vertices, exact_start_timestamp); +} +void Indices::AbortEntries(LabelId label, std::span const> vertices, + uint64_t exact_start_timestamp) const { + static_cast(label_property_index_.get()) + ->AbortEntries(label, vertices, exact_start_timestamp); +} + void Indices::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) const { static_cast(label_index_.get())->RemoveObsoleteEntries(oldest_active_start_timestamp); static_cast(label_property_index_.get()) @@ -50,4 +65,8 @@ Indices::Indices(const Config &config, StorageMode storage_mode) { }); } +Indices::IndexStats Indices::Analysis() const { + return {static_cast(label_index_.get())->Analysis(), + static_cast(label_property_index_.get())->Analysis()}; +} } // namespace memgraph::storage diff --git a/src/storage/v2/indices/indices.hpp b/src/storage/v2/indices/indices.hpp index 9a71107cd..33bd429e6 100644 --- a/src/storage/v2/indices/indices.hpp +++ b/src/storage/v2/indices/indices.hpp @@ -12,6 +12,9 @@ #pragma once #include +#include + +#include "storage/v2/id_types.hpp" #include "storage/v2/indices/label_index.hpp" #include "storage/v2/indices/label_property_index.hpp" #include "storage/v2/storage_mode.hpp" @@ -32,6 +35,20 @@ struct Indices { /// TODO: unused in disk indices void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) const; + /// Surgical removal of entries that was inserted this transaction + /// TODO: unused in disk indices + void AbortEntries(LabelId labelId, std::span vertices, uint64_t exact_start_timestamp) const; + void AbortEntries(PropertyId property, std::span const> vertices, + uint64_t exact_start_timestamp) const; + void AbortEntries(LabelId label, std::span const> vertices, + uint64_t exact_start_timestamp) const; + + struct IndexStats { + std::vector label; + LabelPropertyIndex::IndexStats property_label; + }; + IndexStats Analysis() const; + // Indices are updated whenever an update occurs, instead of only on commit or // advance command. This is necessary because we want indices to support `NEW` // view for use in Merge. diff --git a/src/storage/v2/indices/label_property_index.hpp b/src/storage/v2/indices/label_property_index.hpp index 84908b3e9..e8389fbea 100644 --- a/src/storage/v2/indices/label_property_index.hpp +++ b/src/storage/v2/indices/label_property_index.hpp @@ -19,6 +19,11 @@ namespace memgraph::storage { class LabelPropertyIndex { public: + struct IndexStats { + std::map> l2p; + std::map> p2l; + }; + LabelPropertyIndex() = default; LabelPropertyIndex(const LabelPropertyIndex &) = delete; LabelPropertyIndex(LabelPropertyIndex &&) = delete; diff --git a/src/storage/v2/inmemory/label_index.cpp b/src/storage/v2/inmemory/label_index.cpp index 31c3634be..82ead4b44 100644 --- a/src/storage/v2/inmemory/label_index.cpp +++ b/src/storage/v2/inmemory/label_index.cpp @@ -10,8 +10,12 @@ // licenses/APL.txt. #include "storage/v2/inmemory/label_index.hpp" + +#include + #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/indices/indices_utils.hpp" +#include "storage/v2/inmemory/storage.hpp" namespace memgraph::storage { @@ -96,9 +100,23 @@ void InMemoryLabelIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_time } } -InMemoryLabelIndex::Iterable::Iterable(utils::SkipList::Accessor index_accessor, LabelId label, View view, - Storage *storage, Transaction *transaction) - : index_accessor_(std::move(index_accessor)), +void InMemoryLabelIndex::AbortEntries(LabelId labelId, std::span vertices, + uint64_t exact_start_timestamp) { + auto const it = index_.find(labelId); + if (it == index_.end()) return; + + auto &label_storage = it->second; + auto vertices_acc = label_storage.access(); + for (auto *vertex : vertices) { + vertices_acc.remove(Entry{vertex, exact_start_timestamp}); + } +} + +InMemoryLabelIndex::Iterable::Iterable(utils::SkipList::Accessor index_accessor, + utils::SkipList::ConstAccessor vertices_accessor, LabelId label, + View view, Storage *storage, Transaction *transaction) + : pin_accessor_(std::move(vertices_accessor)), + index_accessor_(std::move(index_accessor)), label_(label), view_(view), storage_(storage), @@ -147,9 +165,21 @@ void InMemoryLabelIndex::RunGC() { InMemoryLabelIndex::Iterable InMemoryLabelIndex::Vertices(LabelId label, View view, Storage *storage, Transaction *transaction) { + DMG_ASSERT(storage->storage_mode_ == StorageMode::IN_MEMORY_TRANSACTIONAL || + storage->storage_mode_ == StorageMode::IN_MEMORY_ANALYTICAL, + "LabelIndex trying to access InMemory vertices from OnDisk!"); + auto vertices_acc = static_cast(storage)->vertices_.access(); const auto it = index_.find(label); MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint()); - return {it->second.access(), label, view, storage, transaction}; + return {it->second.access(), std::move(vertices_acc), label, view, storage, transaction}; +} + +InMemoryLabelIndex::Iterable InMemoryLabelIndex::Vertices( + LabelId label, memgraph::utils::SkipList::ConstAccessor vertices_acc, View view, + Storage *storage, Transaction *transaction) { + const auto it = index_.find(label); + MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint()); + return {it->second.access(), std::move(vertices_acc), label, view, storage, transaction}; } void InMemoryLabelIndex::SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats) { @@ -187,4 +217,12 @@ bool InMemoryLabelIndex::DeleteIndexStats(const storage::LabelId &label) { return false; } +std::vector InMemoryLabelIndex::Analysis() const { + std::vector res; + res.reserve(index_.size()); + for (const auto &[label, _] : index_) { + res.emplace_back(label); + } + return res; +} } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/label_index.hpp b/src/storage/v2/inmemory/label_index.hpp index 7d606574b..21df32deb 100644 --- a/src/storage/v2/inmemory/label_index.hpp +++ b/src/storage/v2/inmemory/label_index.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/indices/label_index.hpp" #include "storage/v2/indices/label_index_stats.hpp" @@ -56,10 +58,15 @@ class InMemoryLabelIndex : public storage::LabelIndex { void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp); + /// Surgical removal of entries that was inserted this transaction + void AbortEntries(LabelId labelId, std::span vertices, uint64_t exact_start_timestamp); + + std::vector Analysis() const; + class Iterable { public: - Iterable(utils::SkipList::Accessor index_accessor, LabelId label, View view, Storage *storage, - Transaction *transaction); + Iterable(utils::SkipList::Accessor index_accessor, utils::SkipList::ConstAccessor vertices_accessor, + LabelId label, View view, Storage *storage, Transaction *transaction); class Iterator { public: @@ -85,6 +92,7 @@ class InMemoryLabelIndex : public storage::LabelIndex { Iterator end() { return {this, index_accessor_.end()}; } private: + utils::SkipList::ConstAccessor pin_accessor_; utils::SkipList::Accessor index_accessor_; LabelId label_; View view_; @@ -98,6 +106,9 @@ class InMemoryLabelIndex : public storage::LabelIndex { Iterable Vertices(LabelId label, View view, Storage *storage, Transaction *transaction); + Iterable Vertices(LabelId label, memgraph::utils::SkipList::ConstAccessor vertices_acc, + View view, Storage *storage, Transaction *transaction); + void SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats); std::optional GetIndexStats(const storage::LabelId &label) const; diff --git a/src/storage/v2/inmemory/label_property_index.cpp b/src/storage/v2/inmemory/label_property_index.cpp index fa5cce444..f61b9dd11 100644 --- a/src/storage/v2/inmemory/label_property_index.cpp +++ b/src/storage/v2/inmemory/label_property_index.cpp @@ -12,6 +12,8 @@ #include "storage/v2/inmemory/label_property_index.hpp" #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/indices/indices_utils.hpp" +#include "storage/v2/inmemory/storage.hpp" +#include "utils/logging.hpp" namespace memgraph::storage { @@ -101,11 +103,12 @@ void InMemoryLabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const return; } - if (!indices_by_property_.contains(property)) { + auto index = indices_by_property_.find(property); + if (index == indices_by_property_.end()) { return; } - for (const auto &[_, storage] : indices_by_property_.at(property)) { + for (const auto &[_, storage] : index->second) { auto acc = storage->access(); acc.insert(Entry{value, vertex, tx.start_timestamp}); } @@ -220,12 +223,14 @@ const PropertyValue kSmallestMap = PropertyValue(std::map(0), std::numeric_limits::min()}); -InMemoryLabelPropertyIndex::Iterable::Iterable(utils::SkipList::Accessor index_accessor, LabelId label, +InMemoryLabelPropertyIndex::Iterable::Iterable(utils::SkipList::Accessor index_accessor, + utils::SkipList::ConstAccessor vertices_accessor, LabelId label, PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view, Storage *storage, Transaction *transaction) - : index_accessor_(std::move(index_accessor)), + : pin_accessor_(std::move(vertices_accessor)), + index_accessor_(std::move(index_accessor)), label_(label), property_(property), lower_bound_(lower_bound), @@ -428,9 +433,57 @@ InMemoryLabelPropertyIndex::Iterable InMemoryLabelPropertyIndex::Vertices( LabelId label, PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view, Storage *storage, Transaction *transaction) { + DMG_ASSERT(storage->storage_mode_ == StorageMode::IN_MEMORY_TRANSACTIONAL || + storage->storage_mode_ == StorageMode::IN_MEMORY_ANALYTICAL, + "PropertyLabel index trying to access InMemory vertices from OnDisk!"); + auto vertices_acc = static_cast(storage)->vertices_.access(); auto it = index_.find({label, property}); MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint()); - return {it->second.access(), label, property, lower_bound, upper_bound, view, storage, transaction}; + return {it->second.access(), std::move(vertices_acc), label, property, lower_bound, upper_bound, view, storage, + transaction}; } +InMemoryLabelPropertyIndex::Iterable InMemoryLabelPropertyIndex::Vertices( + LabelId label, PropertyId property, + memgraph::utils::SkipList::ConstAccessor vertices_acc, + const std::optional> &lower_bound, + const std::optional> &upper_bound, View view, Storage *storage, + Transaction *transaction) { + auto it = index_.find({label, property}); + MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint()); + return {it->second.access(), std::move(vertices_acc), label, property, lower_bound, upper_bound, view, storage, + transaction}; +} + +void InMemoryLabelPropertyIndex::AbortEntries(PropertyId property, + std::span const> vertices, + uint64_t exact_start_timestamp) { + auto const it = indices_by_property_.find(property); + if (it == indices_by_property_.end()) return; + + auto &indices = it->second; + for (const auto &[_, index] : indices) { + auto index_acc = index->access(); + for (auto const &[value, vertex] : vertices) { + index_acc.remove(Entry{value, vertex, exact_start_timestamp}); + } + } +} + +void InMemoryLabelPropertyIndex::AbortEntries(LabelId label, + std::span const> vertices, + uint64_t exact_start_timestamp) { + for (auto &[label_prop, storage] : index_) { + if (label_prop.first != label) { + continue; + } + + auto index_acc = storage.access(); + for (const auto &[property, vertex] : vertices) { + if (!property.IsNull()) { + index_acc.remove(Entry{property, vertex, exact_start_timestamp}); + } + } + } +} } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/label_property_index.hpp b/src/storage/v2/inmemory/label_property_index.hpp index 7f8c54909..ae96a37f8 100644 --- a/src/storage/v2/inmemory/label_property_index.hpp +++ b/src/storage/v2/inmemory/label_property_index.hpp @@ -11,9 +11,13 @@ #pragma once +#include + #include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/id_types.hpp" #include "storage/v2/indices/label_property_index.hpp" #include "storage/v2/indices/label_property_index_stats.hpp" +#include "storage/v2/property_value.hpp" #include "utils/rw_lock.hpp" #include "utils/synchronized.hpp" @@ -61,10 +65,25 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp); + void AbortEntries(PropertyId property, std::span const> vertices, + uint64_t exact_start_timestamp); + void AbortEntries(LabelId label, std::span const> vertices, + uint64_t exact_start_timestamp); + + IndexStats Analysis() const { + IndexStats res{}; + for (const auto &[lp, _] : index_) { + const auto &[label, property] = lp; + res.l2p[label].emplace_back(property); + res.p2l[property].emplace_back(label); + } + return res; + } + class Iterable { public: - Iterable(utils::SkipList::Accessor index_accessor, LabelId label, PropertyId property, - const std::optional> &lower_bound, + Iterable(utils::SkipList::Accessor index_accessor, utils::SkipList::ConstAccessor vertices_accessor, + LabelId label, PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view, Storage *storage, Transaction *transaction); @@ -92,6 +111,7 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { Iterator end(); private: + utils::SkipList::ConstAccessor pin_accessor_; utils::SkipList::Accessor index_accessor_; LabelId label_; PropertyId property_; @@ -131,6 +151,12 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { const std::optional> &upper_bound, View view, Storage *storage, Transaction *transaction); + Iterable Vertices(LabelId label, PropertyId property, + memgraph::utils::SkipList::ConstAccessor vertices_acc, + const std::optional> &lower_bound, + const std::optional> &upper_bound, View view, Storage *storage, + Transaction *transaction); + private: std::map, utils::SkipList> index_; std::unordered_map *>> indices_by_property_; diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index c27018d24..558330996 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -10,21 +10,57 @@ // licenses/APL.txt. #include "storage/v2/inmemory/storage.hpp" +#include +#include #include "dbms/constants.hpp" #include "memory/global_memory_control.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" +#include "storage/v2/edge_direction.hpp" +#include "storage/v2/id_types.hpp" #include "storage/v2/metadata_delta.hpp" /// REPLICATION /// #include "dbms/inmemory/replication_handlers.hpp" #include "storage/v2/inmemory/replication/recovery.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" +#include "storage/v2/property_value.hpp" #include "utils/resource_lock.hpp" #include "utils/stat.hpp" namespace memgraph::storage { +namespace { + +auto FindEdges(const View view, EdgeTypeId edge_type, const VertexAccessor *from_vertex, VertexAccessor *to_vertex) + -> Result { + auto use_out_edges = [](Vertex const *from_vertex, Vertex const *to_vertex) { + // Obtain the locks by `gid` order to avoid lock cycles. + auto guard_from = std::unique_lock{from_vertex->lock, std::defer_lock}; + auto guard_to = std::unique_lock{to_vertex->lock, std::defer_lock}; + if (from_vertex->gid < to_vertex->gid) { + guard_from.lock(); + guard_to.lock(); + } else if (from_vertex->gid > to_vertex->gid) { + guard_to.lock(); + guard_from.lock(); + } else { + // The vertices are the same vertex, only lock one. + guard_from.lock(); + } + + // With the potentially cheaper side FindEdges + const auto out_n = from_vertex->out_edges.size(); + const auto in_n = to_vertex->in_edges.size(); + return out_n <= in_n; + }; + + return use_out_edges(from_vertex->vertex_, to_vertex->vertex_) ? from_vertex->OutEdges(view, {edge_type}, to_vertex) + : to_vertex->InEdges(view, {edge_type}, from_vertex); +} + +}; // namespace + using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) @@ -315,6 +351,24 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_); } +std::optional InMemoryStorage::InMemoryAccessor::FindEdge(Gid gid, const View view, EdgeTypeId edge_type, + VertexAccessor *from_vertex, + VertexAccessor *to_vertex) { + auto res = FindEdges(view, edge_type, from_vertex, to_vertex); + if (res.HasError()) return std::nullopt; // TODO: use a Result type + + auto const it = std::invoke([this, gid, &res]() { + auto const byGid = [gid](EdgeAccessor const &edge_accessor) { return edge_accessor.edge_.gid == gid; }; + auto const byEdgePtr = [gid](EdgeAccessor const &edge_accessor) { return edge_accessor.edge_.ptr->gid == gid; }; + if (config_.properties_on_edges) return std::ranges::find_if(res->edges, byEdgePtr); + return std::ranges::find_if(res->edges, byGid); + }); + + if (it == res->edges.end()) return std::nullopt; // TODO: use a Result type + + return *it; +} + Result InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type, storage::Gid gid) { MG_ASSERT(from->transaction_ == to->transaction_, @@ -697,7 +751,8 @@ utils::BasicResult InMemoryStorage::InMemoryAcce could_replicate_all_sync_replicas = mem_storage->AppendToWalDataDefinition(transaction_, *commit_timestamp_); // protected by engine_guard // TODO: release lock, and update all deltas to have a local copy of the commit timestamp - transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); // protected by engine_guard + transaction_.commit_timestamp->store(*commit_timestamp_, + std::memory_order_release); // protected by engine_guard // Replica can only update the last commit timestamp with // the commits received from main. if (is_main || desired_commit_timestamp.has_value()) { @@ -823,6 +878,21 @@ void InMemoryStorage::InMemoryAccessor::Abort() { std::list my_deleted_vertices; std::list my_deleted_edges; + std::map> label_cleanup; + std::map>> label_property_cleanup; + std::map>> property_cleanup; + + // CONSTRAINTS + if (transaction_.constraint_verification_info.NeedsUniqueConstraintVerification()) { + // Need to remove elements from constraints before handling of the deltas, so the elements match the correct + // values + auto vertices_to_check = transaction_.constraint_verification_info.GetVerticesForUniqueConstraintChecking(); + auto vertices_to_check_v = std::vector{vertices_to_check.begin(), vertices_to_check.end()}; + storage_->constraints_.AbortEntries(vertices_to_check_v, transaction_.start_timestamp); + } + + const auto index_stats = storage_->indices_.Analysis(); + for (const auto &delta : transaction_.deltas.use()) { auto prev = delta.prev.Get(); switch (prev.type) { @@ -838,6 +908,24 @@ void InMemoryStorage::InMemoryAccessor::Abort() { MG_ASSERT(it != vertex->labels.end(), "Invalid database state!"); std::swap(*it, *vertex->labels.rbegin()); vertex->labels.pop_back(); + + // For label index + // check if there is a label index for the label and add entry if so + // For property label index + // check if we care about the label; this will return all the propertyIds we care about and then get + // the current property value + if (std::binary_search(index_stats.label.begin(), index_stats.label.end(), current->label)) { + label_cleanup[current->label].emplace_back(vertex); + } + const auto &properties = index_stats.property_label.l2p.find(current->label); + if (properties != index_stats.property_label.l2p.end()) { + for (const auto &property : properties->second) { + auto current_value = vertex->properties.GetProperty(property); + if (!current_value.IsNull()) { + label_property_cleanup[current->label].emplace_back(std::move(current_value), vertex); + } + } + } break; } case Delta::Action::ADD_LABEL: { @@ -847,6 +935,18 @@ void InMemoryStorage::InMemoryAccessor::Abort() { break; } case Delta::Action::SET_PROPERTY: { + // For label index nothing + // For property label index + // check if we care about the property, this will return all the labels and then get current property + // value + const auto &labels = index_stats.property_label.p2l.find(current->property.key); + if (labels != index_stats.property_label.p2l.end()) { + auto current_value = vertex->properties.GetProperty(current->property.key); + if (!current_value.IsNull()) { + property_cleanup[current->property.key].emplace_back(std::move(current_value), vertex); + } + } + // Setting the correct value vertex->properties.SetProperty(current->property.key, current->property.value); break; } @@ -963,7 +1063,7 @@ void InMemoryStorage::InMemoryAccessor::Abort() { auto *mem_storage = static_cast(storage_); { - std::unique_lock engine_guard(storage_->engine_lock_); + auto engine_guard = std::unique_lock(storage_->engine_lock_); uint64_t mark_timestamp = storage_->timestamp_; // Take garbage_undo_buffers lock while holding the engine lock to make // sure that entries are sorted by mark timestamp in the list. @@ -975,10 +1075,37 @@ void InMemoryStorage::InMemoryAccessor::Abort() { garbage_undo_buffers.emplace_back(mark_timestamp, std::move(transaction_.deltas), std::move(transaction_.commit_timestamp)); }); - mem_storage->deleted_vertices_.WithLock( - [&](auto &deleted_vertices) { deleted_vertices.splice(deleted_vertices.begin(), my_deleted_vertices); }); - mem_storage->deleted_edges_.WithLock( - [&](auto &deleted_edges) { deleted_edges.splice(deleted_edges.begin(), my_deleted_edges); }); + + /// We MUST unlink (aka. remove) entries in indexes and constraints + /// before we unlink (aka. remove) vertices from storage + /// this is because they point into vertices skip_list + + // INDICES + for (auto const &[label, vertices] : label_cleanup) { + storage_->indices_.AbortEntries(label, vertices, transaction_.start_timestamp); + } + for (auto const &[label, prop_vertices] : label_property_cleanup) { + storage_->indices_.AbortEntries(label, prop_vertices, transaction_.start_timestamp); + } + for (auto const &[property, prop_vertices] : property_cleanup) { + storage_->indices_.AbortEntries(property, prop_vertices, transaction_.start_timestamp); + } + + // VERTICES + { + auto vertices_acc = mem_storage->vertices_.access(); + for (auto gid : my_deleted_vertices) { + vertices_acc.remove(gid); + } + } + + // EDGES + { + auto edges_acc = mem_storage->edges_.access(); + for (auto gid : my_deleted_edges) { + edges_acc.remove(gid); + } + } } mem_storage->commit_log_->MarkFinished(transaction_.start_timestamp); @@ -1271,8 +1398,6 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_ // vertices that appear in an index also exist in main storage. std::list current_deleted_edges; std::list current_deleted_vertices; - deleted_vertices_->swap(current_deleted_vertices); - deleted_edges_->swap(current_deleted_edges); auto const need_full_scan_vertices = gc_full_scan_vertices_delete_.exchange(false); auto const need_full_scan_edges = gc_full_scan_edges_delete_.exchange(false); @@ -1922,12 +2047,12 @@ utils::BasicResult InMemoryStorage::Create void InMemoryStorage::FreeMemory(std::unique_lock main_guard) { CollectGarbage(std::move(main_guard)); + static_cast(indices_.label_index_.get())->RunGC(); + static_cast(indices_.label_property_index_.get())->RunGC(); + // SkipList is already threadsafe vertices_.run_gc(); edges_.run_gc(); - - static_cast(indices_.label_index_.get())->RunGC(); - static_cast(indices_.label_property_index_.get())->RunGC(); } uint64_t InMemoryStorage::CommitTimestamp(const std::optional desired_commit_timestamp) { diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 3c50326b2..48d6f0cb7 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -51,6 +51,8 @@ class InMemoryStorage final : public Storage { friend std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker, const InMemoryStorage *storage); + friend class InMemoryLabelIndex; + friend class InMemoryLabelPropertyIndex; public: enum class CreateSnapshotError : uint8_t { DisabledForReplica, ReachedMaxNumTries }; @@ -185,6 +187,9 @@ class InMemoryStorage final : public Storage { /// @throw std::bad_alloc Result CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type) override; + std::optional FindEdge(Gid gid, View view, EdgeTypeId edge_type, VertexAccessor *from_vertex, + VertexAccessor *to_vertex) override; + Result EdgeSetFrom(EdgeAccessor *edge, VertexAccessor *new_from) override; Result EdgeSetTo(EdgeAccessor *edge, VertexAccessor *new_to) override; diff --git a/src/storage/v2/inmemory/unique_constraints.cpp b/src/storage/v2/inmemory/unique_constraints.cpp index 78929bc38..6a2945883 100644 --- a/src/storage/v2/inmemory/unique_constraints.cpp +++ b/src/storage/v2/inmemory/unique_constraints.cpp @@ -256,11 +256,12 @@ bool InMemoryUniqueConstraints::Entry::operator==(const std::vectorlabels) { - if (!constraints_by_label_.contains(label)) { + const auto &constraint = constraints_by_label_.find(label); + if (constraint == constraints_by_label_.end()) { continue; } - for (auto &[props, storage] : constraints_by_label_.at(label)) { + for (auto &[props, storage] : constraint->second) { auto values = vertex->properties.ExtractPropertyValues(props); if (!values) { @@ -273,6 +274,28 @@ void InMemoryUniqueConstraints::UpdateBeforeCommit(const Vertex *vertex, const T } } +void InMemoryUniqueConstraints::AbortEntries(std::span vertices, uint64_t exact_start_timestamp) { + for (const auto &vertex : vertices) { + for (const auto &label : vertex->labels) { + const auto &constraint = constraints_by_label_.find(label); + if (constraint == constraints_by_label_.end()) { + return; + } + + for (auto &[props, storage] : constraint->second) { + auto values = vertex->properties.ExtractPropertyValues(props); + + if (!values) { + continue; + } + + auto acc = storage->access(); + acc.remove(Entry{std::move(*values), vertex, exact_start_timestamp}); + } + } + } +} + utils::BasicResult InMemoryUniqueConstraints::CreateConstraint(LabelId label, const std::set &properties, utils::SkipList::Accessor vertices) { @@ -364,12 +387,14 @@ std::optional InMemoryUniqueConstraints::Validate(const Ver if (vertex.deleted) { return std::nullopt; } + for (const auto &label : vertex.labels) { - if (!constraints_by_label_.contains(label)) { + const auto &constraint = constraints_by_label_.find(label); + if (constraint == constraints_by_label_.end()) { continue; } - for (const auto &[properties, storage] : constraints_by_label_.at(label)) { + for (const auto &[properties, storage] : constraint->second) { auto value_array = vertex.properties.ExtractPropertyValues(properties); if (!value_array) { diff --git a/src/storage/v2/inmemory/unique_constraints.hpp b/src/storage/v2/inmemory/unique_constraints.hpp index 45472ca74..d1e590357 100644 --- a/src/storage/v2/inmemory/unique_constraints.hpp +++ b/src/storage/v2/inmemory/unique_constraints.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "storage/v2/constraints/unique_constraints.hpp" namespace memgraph::storage { @@ -54,6 +56,8 @@ class InMemoryUniqueConstraints : public UniqueConstraints { void UpdateBeforeCommit(const Vertex *vertex, std::unordered_set &added_labels, std::unordered_set &added_properties, const Transaction &tx); + void AbortEntries(std::span vertices, uint64_t exact_start_timestamp); + /// Creates unique constraint on the given `label` and a list of `properties`. /// Returns constraint violation if there are multiple vertices with the same /// label and property values. Returns `CreationStatus::ALREADY_EXISTS` if diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 4142da3ca..60752d101 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -195,6 +195,9 @@ class Storage { virtual Result CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type) = 0; + virtual std::optional FindEdge(Gid gid, View view, EdgeTypeId edge_type, VertexAccessor *from_vertex, + VertexAccessor *to_vertex) = 0; + virtual Result EdgeSetFrom(EdgeAccessor *edge, VertexAccessor *new_from) = 0; virtual Result EdgeSetTo(EdgeAccessor *edge, VertexAccessor *new_to) = 0; diff --git a/tests/e2e/concurrent_query_modules/test_query_modules/module_test.cpp b/tests/e2e/concurrent_query_modules/test_query_modules/module_test.cpp index 44479d900..b53dda881 100644 --- a/tests/e2e/concurrent_query_modules/test_query_modules/module_test.cpp +++ b/tests/e2e/concurrent_query_modules/test_query_modules/module_test.cpp @@ -16,11 +16,11 @@ #include #include -constexpr char *kProcedureHackerNews = "hacker_news"; -constexpr char *kArgumentHackerNewsVotes = "votes"; -constexpr char *kArgumentHackerNewsItemHourAge = "item_hour_age"; -constexpr char *kArgumentHackerNewsGravity = "gravity"; -constexpr char *kReturnHackerNewsScore = "score"; +constexpr char const *kProcedureHackerNews = "hacker_news"; +constexpr char const *kArgumentHackerNewsVotes = "votes"; +constexpr char const *kArgumentHackerNewsItemHourAge = "item_hour_age"; +constexpr char const *kArgumentHackerNewsGravity = "gravity"; +constexpr char const *kReturnHackerNewsScore = "score"; void HackerNews(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard(memory); diff --git a/tests/e2e/replication/CMakeLists.txt b/tests/e2e/replication/CMakeLists.txt index b054981c8..39f179a3d 100644 --- a/tests/e2e/replication/CMakeLists.txt +++ b/tests/e2e/replication/CMakeLists.txt @@ -13,6 +13,7 @@ copy_e2e_python_files(replication_show common.py) copy_e2e_python_files(replication_show conftest.py) copy_e2e_python_files(replication_show show.py) copy_e2e_python_files(replication_show show_while_creating_invalid_state.py) +copy_e2e_python_files(replication_show edge_delete.py) copy_e2e_python_files_from_parent_folder(replication_show ".." memgraph.py) copy_e2e_python_files_from_parent_folder(replication_show ".." interactive_mg_runner.py) copy_e2e_python_files_from_parent_folder(replication_show ".." mg_utils.py) diff --git a/tests/e2e/replication/edge_delete.py b/tests/e2e/replication/edge_delete.py new file mode 100755 index 000000000..0e25faee1 --- /dev/null +++ b/tests/e2e/replication/edge_delete.py @@ -0,0 +1,56 @@ +# Copyright 2022 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys +import time + +import pytest +from common import execute_and_fetch_all +from mg_utils import mg_sleep_and_assert + + +# BUGFIX: for issue https://github.com/memgraph/memgraph/issues/1515 +def test_replication_handles_delete_when_multiple_edges_of_same_type(connection): + # Goal is to check the timestamp are correctly computed from the information we get from replicas. + # 0/ Check original state of replicas. + # 1/ Add nodes and edges to MAIN, then delete the edges. + # 2/ Check state of replicas. + + # 0/ + conn = connection(7687, "main") + conn.autocommit = True + cursor = conn.cursor() + actual_data = set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "async", 0, 0, "ready"), + } + assert actual_data == expected_data + + # 1/ + execute_and_fetch_all(cursor, "CREATE (a)-[r:X]->(b) CREATE (a)-[:X]->(b) DELETE r;") + + # 2/ + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 2, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "async", 2, 0, "ready"), + } + + def retrieve_data(): + return set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/replication/workloads.yaml b/tests/e2e/replication/workloads.yaml index 72269d652..fc239b221 100644 --- a/tests/e2e/replication/workloads.yaml +++ b/tests/e2e/replication/workloads.yaml @@ -8,6 +8,23 @@ template_validation_queries: &template_validation_queries validation_queries: - <<: *template_test_nodes_query - <<: *template_test_edges_query +template_simple_cluster: &template_simple_cluster + cluster: + replica_1: + args: [ "--bolt-port", "7688", "--log-level=TRACE" ] + log_file: "replication-e2e-replica1.log" + setup_queries: [ "SET REPLICATION ROLE TO REPLICA WITH PORT 10001;" ] + replica_2: + args: ["--bolt-port", "7689", "--log-level=TRACE"] + log_file: "replication-e2e-replica2.log" + setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"] + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "replication-e2e-main.log" + setup_queries: [ + "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001'", + "REGISTER REPLICA replica_2 ASYNC TO '127.0.0.1:10002'", + ] template_cluster: &template_cluster cluster: replica_1: @@ -83,3 +100,8 @@ workloads: - name: "Show while creating invalid state" binary: "tests/e2e/pytest_runner.sh" args: ["replication/show_while_creating_invalid_state.py"] + + - name: "Delete edge replication" + binary: "tests/e2e/pytest_runner.sh" + args: ["replication/edge_delete.py"] + <<: *template_simple_cluster From 3ccd78ac71d423fd15ca55998a766c8d08cdaa46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Pu=C5=A1i=C4=87?= Date: Sat, 2 Dec 2023 20:03:40 +0100 Subject: [PATCH 33/52] Add path and weight to variable expand filter (#1434) Co-authored-by: Aidar Samerkhanov --- src/query/frontend/ast/ast.hpp | 6 + .../frontend/ast/cypher_main_visitor.cpp | 30 ++++ .../frontend/opencypher/grammar/Cypher.g4 | 2 +- .../frontend/semantic/symbol_generator.cpp | 23 ++- src/query/plan/operator.cpp | 167 ++++++++++++------ src/query/plan/operator.hpp | 6 + src/query/plan/preprocess.cpp | 14 ++ src/query/plan/rewrite/index_lookup.hpp | 5 + src/query/plan/rule_based_planner.hpp | 53 +++++- src/query/plan/variable_start_planner.cpp | 3 +- .../tests/memgraph_V1/features/match.feature | 72 ++++++++ .../features/memgraph_allshortest.feature | 100 +++++++++++ .../memgraph_V1/features/memgraph_bfs.feature | 92 ++++++++++ .../features/memgraph_wshortest.feature | 100 +++++++++++ .../memgraph_V1/graphs/graph_edges.cypher | 2 + .../memgraph_V1/graphs/graph_index.cypher | 2 + tests/unit/cypher_main_visitor.cpp | 86 +++++++++ 17 files changed, 692 insertions(+), 71 deletions(-) create mode 100644 tests/gql_behave/tests/memgraph_V1/graphs/graph_edges.cypher create mode 100644 tests/gql_behave/tests/memgraph_V1/graphs/graph_index.cypher diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index d63736c85..a36f1a8b5 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -1818,6 +1818,10 @@ class EdgeAtom : public memgraph::query::PatternAtom { memgraph::query::Identifier *inner_edge{nullptr}; /// Argument identifier for the destination node of the edge. memgraph::query::Identifier *inner_node{nullptr}; + /// Argument identifier for the currently-accumulated path. + memgraph::query::Identifier *accumulated_path{nullptr}; + /// Argument identifier for the weight of the currently-accumulated path. + memgraph::query::Identifier *accumulated_weight{nullptr}; /// Evaluates the result of the lambda. memgraph::query::Expression *expression{nullptr}; @@ -1825,6 +1829,8 @@ class EdgeAtom : public memgraph::query::PatternAtom { Lambda object; object.inner_edge = inner_edge ? inner_edge->Clone(storage) : nullptr; object.inner_node = inner_node ? inner_node->Clone(storage) : nullptr; + object.accumulated_path = accumulated_path ? accumulated_path->Clone(storage) : nullptr; + object.accumulated_weight = accumulated_weight ? accumulated_weight->Clone(storage) : nullptr; object.expression = expression ? expression->Clone(storage) : nullptr; return object; } diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 4bf7f36fd..cf9709a31 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -1978,6 +1978,15 @@ antlrcpp::Any CypherMainVisitor::visitRelationshipPattern(MemgraphCypher::Relati edge_lambda.inner_edge = storage_->Create(traversed_edge_variable); auto traversed_node_variable = std::any_cast(lambda->traversed_node->accept(this)); edge_lambda.inner_node = storage_->Create(traversed_node_variable); + if (lambda->accumulated_path) { + auto accumulated_path_variable = std::any_cast(lambda->accumulated_path->accept(this)); + edge_lambda.accumulated_path = storage_->Create(accumulated_path_variable); + + if (lambda->accumulated_weight) { + auto accumulated_weight_variable = std::any_cast(lambda->accumulated_weight->accept(this)); + edge_lambda.accumulated_weight = storage_->Create(accumulated_weight_variable); + } + } edge_lambda.expression = std::any_cast(lambda->expression()->accept(this)); return edge_lambda; }; @@ -2002,6 +2011,15 @@ antlrcpp::Any CypherMainVisitor::visitRelationshipPattern(MemgraphCypher::Relati // In variable expansion inner variables are mandatory. anonymous_identifiers.push_back(&edge->filter_lambda_.inner_edge); anonymous_identifiers.push_back(&edge->filter_lambda_.inner_node); + + // TODO: In what use case do we need accumulated path and weight here? + if (edge->filter_lambda_.accumulated_path) { + anonymous_identifiers.push_back(&edge->filter_lambda_.accumulated_path); + + if (edge->filter_lambda_.accumulated_weight) { + anonymous_identifiers.push_back(&edge->filter_lambda_.accumulated_weight); + } + } break; case 1: if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || @@ -2013,9 +2031,21 @@ antlrcpp::Any CypherMainVisitor::visitRelationshipPattern(MemgraphCypher::Relati // Add mandatory inner variables for filter lambda. anonymous_identifiers.push_back(&edge->filter_lambda_.inner_edge); anonymous_identifiers.push_back(&edge->filter_lambda_.inner_node); + if (edge->filter_lambda_.accumulated_path) { + anonymous_identifiers.push_back(&edge->filter_lambda_.accumulated_path); + + if (edge->filter_lambda_.accumulated_weight) { + anonymous_identifiers.push_back(&edge->filter_lambda_.accumulated_weight); + } + } } else { // Other variable expands only have the filter lambda. edge->filter_lambda_ = visit_lambda(relationshipLambdas[0]); + if (edge->filter_lambda_.accumulated_weight) { + throw SemanticException( + "Accumulated weight in filter lambda can be used only with " + "shortest paths expansion."); + } } break; case 2: diff --git a/src/query/frontend/opencypher/grammar/Cypher.g4 b/src/query/frontend/opencypher/grammar/Cypher.g4 index 53f1fc765..783613695 100644 --- a/src/query/frontend/opencypher/grammar/Cypher.g4 +++ b/src/query/frontend/opencypher/grammar/Cypher.g4 @@ -175,7 +175,7 @@ relationshipDetail : '[' ( name=variable )? ( relationshipTypes )? ( variableExp | '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? relationshipLambda ( total_weight=variable )? (relationshipLambda )? ']' | '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? (properties )* ( relationshipLambda total_weight=variable )? (relationshipLambda )? ']'; -relationshipLambda: '(' traversed_edge=variable ',' traversed_node=variable '|' expression ')'; +relationshipLambda: '(' traversed_edge=variable ',' traversed_node=variable ( ',' accumulated_path=variable )? ( ',' accumulated_weight=variable )? '|' expression ')'; variableExpansion : '*' (BFS | WSHORTEST | ALLSHORTEST)? ( expression )? ( '..' ( expression )? )? ; diff --git a/src/query/frontend/semantic/symbol_generator.cpp b/src/query/frontend/semantic/symbol_generator.cpp index 30790ee4e..a3e855301 100644 --- a/src/query/frontend/semantic/symbol_generator.cpp +++ b/src/query/frontend/semantic/symbol_generator.cpp @@ -658,8 +658,16 @@ bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) { scope.in_edge_range = false; scope.in_pattern = false; if (edge_atom.filter_lambda_.expression) { - VisitWithIdentifiers(edge_atom.filter_lambda_.expression, - {edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node}); + std::vector filter_lambda_identifiers{edge_atom.filter_lambda_.inner_edge, + edge_atom.filter_lambda_.inner_node}; + if (edge_atom.filter_lambda_.accumulated_path) { + filter_lambda_identifiers.emplace_back(edge_atom.filter_lambda_.accumulated_path); + + if (edge_atom.filter_lambda_.accumulated_weight) { + filter_lambda_identifiers.emplace_back(edge_atom.filter_lambda_.accumulated_weight); + } + } + VisitWithIdentifiers(edge_atom.filter_lambda_.expression, filter_lambda_identifiers); } else { // Create inner symbols, but don't bind them in scope, since they are to // be used in the missing filter expression. @@ -668,6 +676,17 @@ bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) { auto *inner_node = edge_atom.filter_lambda_.inner_node; inner_node->MapTo( symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX)); + if (edge_atom.filter_lambda_.accumulated_path) { + auto *accumulated_path = edge_atom.filter_lambda_.accumulated_path; + accumulated_path->MapTo( + symbol_table_->CreateSymbol(accumulated_path->name_, accumulated_path->user_declared_, Symbol::Type::PATH)); + + if (edge_atom.filter_lambda_.accumulated_weight) { + auto *accumulated_weight = edge_atom.filter_lambda_.accumulated_weight; + accumulated_weight->MapTo(symbol_table_->CreateSymbol( + accumulated_weight->name_, accumulated_weight->user_declared_, Symbol::Type::NUMBER)); + } + } } if (edge_atom.weight_lambda_.expression) { VisitWithIdentifiers(edge_atom.weight_lambda_.expression, diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 63bf5cd40..24ce66a69 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -1138,6 +1138,11 @@ class ExpandVariableCursor : public Cursor { edges_it_.emplace_back(edges_.back().begin()); } + if (self_.filter_lambda_.accumulated_path_symbol) { + // Add initial vertex of path to the accumulated path + frame[self_.filter_lambda_.accumulated_path_symbol.value()] = Path(vertex); + } + // reset the frame value to an empty edge list auto *pull_memory = context.evaluation_context.memory; frame[self_.common_.edge_symbol] = TypedValue::TVector(pull_memory); @@ -1234,6 +1239,13 @@ class ExpandVariableCursor : public Cursor { // Skip expanding out of filtered expansion. frame[self_.filter_lambda_.inner_edge_symbol] = current_edge.first; frame[self_.filter_lambda_.inner_node_symbol] = current_vertex; + if (self_.filter_lambda_.accumulated_path_symbol) { + MG_ASSERT(frame[self_.filter_lambda_.accumulated_path_symbol.value()].IsPath(), + "Accumulated path must be path"); + Path &accumulated_path = frame[self_.filter_lambda_.accumulated_path_symbol.value()].ValuePath(); + accumulated_path.Expand(current_edge.first); + accumulated_path.Expand(current_vertex); + } if (self_.filter_lambda_.expression && !EvaluateFilter(evaluator, self_.filter_lambda_.expression)) continue; // we are doing depth-first search, so place the current @@ -1546,6 +1558,13 @@ class SingleSourceShortestPathCursor : public query::plan::Cursor { #endif frame[self_.filter_lambda_.inner_edge_symbol] = edge; frame[self_.filter_lambda_.inner_node_symbol] = vertex; + if (self_.filter_lambda_.accumulated_path_symbol) { + MG_ASSERT(frame[self_.filter_lambda_.accumulated_path_symbol.value()].IsPath(), + "Accumulated path must have Path type"); + Path &accumulated_path = frame[self_.filter_lambda_.accumulated_path_symbol.value()].ValuePath(); + accumulated_path.Expand(edge); + accumulated_path.Expand(vertex); + } if (self_.filter_lambda_.expression) { TypedValue result = self_.filter_lambda_.expression->Accept(evaluator); @@ -1607,6 +1626,11 @@ class SingleSourceShortestPathCursor : public query::plan::Cursor { const auto &vertex = vertex_value.ValueVertex(); processed_.emplace(vertex, std::nullopt); + if (self_.filter_lambda_.accumulated_path_symbol) { + // Add initial vertex of path to the accumulated path + frame[self_.filter_lambda_.accumulated_path_symbol.value()] = Path(vertex); + } + expand_from_vertex(vertex); // go back to loop start and see if we expanded anything @@ -1677,6 +1701,10 @@ class SingleSourceShortestPathCursor : public query::plan::Cursor { namespace { void CheckWeightType(TypedValue current_weight, utils::MemoryResource *memory) { + if (current_weight.IsNull()) { + return; + } + if (!current_weight.IsNumeric() && !current_weight.IsDuration()) { throw QueryRuntimeException("Calculated weight must be numeric or a Duration, got {}.", current_weight.type()); } @@ -1694,6 +1722,34 @@ void CheckWeightType(TypedValue current_weight, utils::MemoryResource *memory) { } } +void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) { + if ((lhs.IsNumeric() && rhs.IsNumeric()) || (lhs.IsDuration() && rhs.IsDuration())) { + return; + } + throw QueryRuntimeException(utils::MessageWithLink( + "All weights should be of the same type, either numeric or a Duration. Please update the weight " + "expression or the filter expression.", + "https://memgr.ph/wsp")); +} + +TypedValue CalculateNextWeight(const std::optional &weight_lambda, + const TypedValue &total_weight, ExpressionEvaluator evaluator) { + if (!weight_lambda) { + return {}; + } + auto *memory = evaluator.GetMemoryResource(); + TypedValue current_weight = weight_lambda->expression->Accept(evaluator); + CheckWeightType(current_weight, memory); + + if (total_weight.IsNull()) { + return current_weight; + } + + ValidateWeightTypes(current_weight, total_weight); + + return TypedValue(current_weight, memory) + total_weight; +} + } // namespace class ExpandWeightedShortestPathCursor : public query::plan::Cursor { @@ -1722,7 +1778,6 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { auto expand_pair = [this, &evaluator, &frame, &create_state, &context]( const EdgeAccessor &edge, const VertexAccessor &vertex, const TypedValue &total_weight, int64_t depth) { - auto *memory = evaluator.GetMemoryResource(); #ifdef MG_ENTERPRISE if (license::global_license_checker.IsEnterpriseValidFast() && context.auth_checker && !(context.auth_checker->Has(vertex, storage::View::OLD, @@ -1731,32 +1786,31 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { return; } #endif + + frame[self_.weight_lambda_->inner_edge_symbol] = edge; + frame[self_.weight_lambda_->inner_node_symbol] = vertex; + TypedValue next_weight = CalculateNextWeight(self_.weight_lambda_, total_weight, evaluator); + if (self_.filter_lambda_.expression) { frame[self_.filter_lambda_.inner_edge_symbol] = edge; frame[self_.filter_lambda_.inner_node_symbol] = vertex; + if (self_.filter_lambda_.accumulated_path_symbol) { + MG_ASSERT(frame[self_.filter_lambda_.accumulated_path_symbol.value()].IsPath(), + "Accumulated path must be path"); + Path &accumulated_path = frame[self_.filter_lambda_.accumulated_path_symbol.value()].ValuePath(); + accumulated_path.Expand(edge); + accumulated_path.Expand(vertex); + + if (self_.filter_lambda_.accumulated_weight_symbol) { + frame[self_.filter_lambda_.accumulated_weight_symbol.value()] = next_weight; + } + } if (!EvaluateFilter(evaluator, self_.filter_lambda_.expression)) return; } - frame[self_.weight_lambda_->inner_edge_symbol] = edge; - frame[self_.weight_lambda_->inner_node_symbol] = vertex; - - TypedValue current_weight = self_.weight_lambda_->expression->Accept(evaluator); - - CheckWeightType(current_weight, memory); - auto next_state = create_state(vertex, depth); - TypedValue next_weight = std::invoke([&] { - if (total_weight.IsNull()) { - return current_weight; - } - - ValidateWeightTypes(current_weight, total_weight); - - return TypedValue(current_weight, memory) + total_weight; - }); - auto found_it = total_cost_.find(next_state); if (found_it != total_cost_.end() && (found_it->second.IsNull() || (found_it->second <= next_weight).ValueBool())) return; @@ -1796,6 +1850,10 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { // Skip expansion for such nodes. if (node.IsNull()) continue; } + if (self_.filter_lambda_.accumulated_path_symbol) { + // Add initial vertex of path to the accumulated path + frame[self_.filter_lambda_.accumulated_path_symbol.value()] = Path(vertex); + } if (self_.upper_bound_) { upper_bound_ = EvaluateInt(&evaluator, self_.upper_bound_, "Max depth in weighted shortest path expansion"); upper_bound_set_ = true; @@ -1808,12 +1866,17 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { "Maximum depth in weighted shortest path expansion must be at " "least 1."); + frame[self_.weight_lambda_->inner_edge_symbol] = TypedValue(); + frame[self_.weight_lambda_->inner_node_symbol] = vertex; + TypedValue current_weight = + CalculateNextWeight(self_.weight_lambda_, /* total_weight */ TypedValue(), evaluator); + // Clear existing data structures. previous_.clear(); total_cost_.clear(); yielded_vertices_.clear(); - pq_.emplace(TypedValue(), 0, vertex, std::nullopt); + pq_.emplace(current_weight, 0, vertex, std::nullopt); // We are adding the starting vertex to the set of yielded vertices // because we don't want to yield paths that end with the starting // vertex. @@ -1913,15 +1976,6 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { // Keeps track of vertices for which we yielded a path already. utils::pmr::unordered_set yielded_vertices_; - static void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) { - if (!((lhs.IsNumeric() && lhs.IsNumeric()) || (rhs.IsDuration() && rhs.IsDuration()))) { - throw QueryRuntimeException(utils::MessageWithLink( - "All weights should be of the same type, either numeric or a Duration. Please update the weight " - "expression or the filter expression.", - "https://memgr.ph/wsp")); - } - } - // Priority queue comparator. Keep lowest weight on top of the queue. class PriorityQueueComparator { public: @@ -1979,36 +2033,32 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor { // queue. auto expand_vertex = [this, &evaluator, &frame](const EdgeAccessor &edge, const EdgeAtom::Direction direction, const TypedValue &total_weight, int64_t depth) { - auto *memory = evaluator.GetMemoryResource(); - auto const &next_vertex = direction == EdgeAtom::Direction::IN ? edge.From() : edge.To(); + // Evaluate current weight + frame[self_.weight_lambda_->inner_edge_symbol] = edge; + frame[self_.weight_lambda_->inner_node_symbol] = next_vertex; + TypedValue next_weight = CalculateNextWeight(self_.weight_lambda_, total_weight, evaluator); + // If filter expression exists, evaluate filter if (self_.filter_lambda_.expression) { frame[self_.filter_lambda_.inner_edge_symbol] = edge; frame[self_.filter_lambda_.inner_node_symbol] = next_vertex; + if (self_.filter_lambda_.accumulated_path_symbol) { + MG_ASSERT(frame[self_.filter_lambda_.accumulated_path_symbol.value()].IsPath(), + "Accumulated path must be path"); + Path &accumulated_path = frame[self_.filter_lambda_.accumulated_path_symbol.value()].ValuePath(); + accumulated_path.Expand(edge); + accumulated_path.Expand(next_vertex); + + if (self_.filter_lambda_.accumulated_weight_symbol) { + frame[self_.filter_lambda_.accumulated_weight_symbol.value()] = next_weight; + } + } if (!EvaluateFilter(evaluator, self_.filter_lambda_.expression)) return; } - // Evaluate current weight - frame[self_.weight_lambda_->inner_edge_symbol] = edge; - frame[self_.weight_lambda_->inner_node_symbol] = next_vertex; - - TypedValue current_weight = self_.weight_lambda_->expression->Accept(evaluator); - - CheckWeightType(current_weight, memory); - - TypedValue next_weight = std::invoke([&] { - if (total_weight.IsNull()) { - return current_weight; - } - - ValidateWeightTypes(current_weight, total_weight); - - return TypedValue(current_weight, memory) + total_weight; - }); - auto found_it = visited_cost_.find(next_vertex); // Check if the vertex has already been processed. if (found_it != visited_cost_.end()) { @@ -2200,7 +2250,17 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor { traversal_stack_.clear(); total_cost_.clear(); - expand_from_vertex(*start_vertex, TypedValue(), 0); + if (self_.filter_lambda_.accumulated_path_symbol) { + // Add initial vertex of path to the accumulated path + frame[self_.filter_lambda_.accumulated_path_symbol.value()] = Path(*start_vertex); + } + + frame[self_.weight_lambda_->inner_edge_symbol] = TypedValue(); + frame[self_.weight_lambda_->inner_node_symbol] = *start_vertex; + TypedValue current_weight = + CalculateNextWeight(self_.weight_lambda_, /* total_weight */ TypedValue(), evaluator); + + expand_from_vertex(*start_vertex, current_weight, 0); visited_cost_.emplace(*start_vertex, 0); frame[self_.common_.edge_symbol] = TypedValue::TVector(memory); } @@ -2252,15 +2312,6 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor { // Stack indicating the traversal level. utils::pmr::list> traversal_stack_; - static void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) { - if (!((lhs.IsNumeric() && lhs.IsNumeric()) || (rhs.IsDuration() && rhs.IsDuration()))) { - throw QueryRuntimeException(utils::MessageWithLink( - "All weights should be of the same type, either numeric or a Duration. Please update the weight " - "expression or the filter expression.", - "https://memgr.ph/wsp")); - } - } - // Priority queue comparator. Keep lowest weight on top of the queue. class PriorityQueueComparator { public: diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index 03df07378..8fa3d3a7c 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -917,12 +917,18 @@ struct ExpansionLambda { Symbol inner_node_symbol; /// Expression used in lambda during expansion. Expression *expression; + /// Currently expanded accumulated path symbol. + std::optional accumulated_path_symbol; + /// Currently expanded accumulated weight symbol. + std::optional accumulated_weight_symbol; ExpansionLambda Clone(AstStorage *storage) const { ExpansionLambda object; object.inner_edge_symbol = inner_edge_symbol; object.inner_node_symbol = inner_node_symbol; object.expression = expression ? expression->Clone(storage) : nullptr; + object.accumulated_path_symbol = accumulated_path_symbol; + object.accumulated_weight_symbol = accumulated_weight_symbol; return object; } }; diff --git a/src/query/plan/preprocess.cpp b/src/query/plan/preprocess.cpp index e03c51841..22899cbc0 100644 --- a/src/query/plan/preprocess.cpp +++ b/src/query/plan/preprocess.cpp @@ -74,6 +74,13 @@ std::vector NormalizePatterns(const SymbolTable &symbol_table, const // Remove symbols which are bound by lambda arguments. collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_edge)); collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_node)); + if (edge->filter_lambda_.accumulated_path) { + collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.accumulated_path)); + + if (edge->filter_lambda_.accumulated_weight) { + collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.accumulated_weight)); + } + } if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || edge->type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS) { collector.symbols_.erase(symbol_table.at(*edge->weight_lambda_.inner_edge)); @@ -295,6 +302,13 @@ void Filters::CollectPatternFilters(Pattern &pattern, SymbolTable &symbol_table, prop_pair.second->Accept(collector); collector.symbols_.emplace(symbol_table.at(*atom->filter_lambda_.inner_node)); collector.symbols_.emplace(symbol_table.at(*atom->filter_lambda_.inner_edge)); + if (atom->filter_lambda_.accumulated_path) { + collector.symbols_.emplace(symbol_table.at(*atom->filter_lambda_.accumulated_path)); + + if (atom->filter_lambda_.accumulated_weight) { + collector.symbols_.emplace(symbol_table.at(*atom->filter_lambda_.accumulated_weight)); + } + } // First handle the inline property filter. auto *property_lookup = storage.Create(atom->filter_lambda_.inner_edge, prop_pair.first); auto *prop_equal = storage.Create(property_lookup, prop_pair.second); diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 4054f8c12..7bb88f659 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -171,6 +171,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { if (expand.common_.existing_node) { return true; } + if (expand.type_ == EdgeAtom::Type::BREADTH_FIRST && expand.filter_lambda_.accumulated_path_symbol) { + // When accumulated path is used, we cannot use ST shortest path algorithm. + return false; + } + std::unique_ptr indexed_scan; ScanAll dst_scan(expand.input(), expand.common_.node_symbol, storage::View::OLD); // With expand to existing we only get real gains with BFS, because we use a diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index bdac76a93..b58d0170b 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -705,9 +705,9 @@ class RuleBasedPlanner { std::optional total_weight; if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || edge->type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS) { - weight_lambda.emplace(ExpansionLambda{symbol_table.at(*edge->weight_lambda_.inner_edge), - symbol_table.at(*edge->weight_lambda_.inner_node), - edge->weight_lambda_.expression}); + weight_lambda.emplace(ExpansionLambda{.inner_edge_symbol = symbol_table.at(*edge->weight_lambda_.inner_edge), + .inner_node_symbol = symbol_table.at(*edge->weight_lambda_.inner_node), + .expression = edge->weight_lambda_.expression}); total_weight.emplace(symbol_table.at(*edge->total_weight_)); } @@ -715,12 +715,28 @@ class RuleBasedPlanner { ExpansionLambda filter_lambda; filter_lambda.inner_edge_symbol = symbol_table.at(*edge->filter_lambda_.inner_edge); filter_lambda.inner_node_symbol = symbol_table.at(*edge->filter_lambda_.inner_node); + if (edge->filter_lambda_.accumulated_path) { + filter_lambda.accumulated_path_symbol = symbol_table.at(*edge->filter_lambda_.accumulated_path); + + if (edge->filter_lambda_.accumulated_weight) { + filter_lambda.accumulated_weight_symbol = symbol_table.at(*edge->filter_lambda_.accumulated_weight); + } + } { // Bind the inner edge and node symbols so they're available for // inline filtering in ExpandVariable. bool inner_edge_bound = bound_symbols.insert(filter_lambda.inner_edge_symbol).second; bool inner_node_bound = bound_symbols.insert(filter_lambda.inner_node_symbol).second; MG_ASSERT(inner_edge_bound && inner_node_bound, "An inner edge and node can't be bound from before"); + if (filter_lambda.accumulated_path_symbol) { + bool accumulated_path_bound = bound_symbols.insert(*filter_lambda.accumulated_path_symbol).second; + MG_ASSERT(accumulated_path_bound, "The accumulated path can't be bound from before"); + + if (filter_lambda.accumulated_weight_symbol) { + bool accumulated_weight_bound = bound_symbols.insert(*filter_lambda.accumulated_weight_symbol).second; + MG_ASSERT(accumulated_weight_bound, "The accumulated weight can't be bound from before"); + } + } } // Join regular filters with lambda filter expression, so that they // are done inline together. Semantic analysis should guarantee that @@ -731,15 +747,34 @@ class RuleBasedPlanner { // filtering (they use the inner symbols. If they were not collected, // we have to remove them manually because no other filter-extraction // will ever bind them again. - filters.erase( - std::remove_if(filters.begin(), filters.end(), - [e = filter_lambda.inner_edge_symbol, n = filter_lambda.inner_node_symbol](FilterInfo &fi) { - return utils::Contains(fi.used_symbols, e) || utils::Contains(fi.used_symbols, n); - }), - filters.end()); + std::vector inner_symbols = {filter_lambda.inner_edge_symbol, filter_lambda.inner_node_symbol}; + if (filter_lambda.accumulated_path_symbol) { + inner_symbols.emplace_back(*filter_lambda.accumulated_path_symbol); + + if (filter_lambda.accumulated_weight_symbol) { + inner_symbols.emplace_back(*filter_lambda.accumulated_weight_symbol); + } + } + + filters.erase(std::remove_if(filters.begin(), filters.end(), + [&inner_symbols](FilterInfo &fi) { + for (const auto &symbol : inner_symbols) { + if (utils::Contains(fi.used_symbols, symbol)) return true; + } + return false; + }), + filters.end()); + // Unbind the temporarily bound inner symbols for filtering. bound_symbols.erase(filter_lambda.inner_edge_symbol); bound_symbols.erase(filter_lambda.inner_node_symbol); + if (filter_lambda.accumulated_path_symbol) { + bound_symbols.erase(*filter_lambda.accumulated_path_symbol); + + if (filter_lambda.accumulated_weight_symbol) { + bound_symbols.erase(*filter_lambda.accumulated_weight_symbol); + } + } if (total_weight) { bound_symbols.insert(*total_weight); diff --git a/src/query/plan/variable_start_planner.cpp b/src/query/plan/variable_start_planner.cpp index 1c230628a..4aa3580d0 100644 --- a/src/query/plan/variable_start_planner.cpp +++ b/src/query/plan/variable_start_planner.cpp @@ -72,8 +72,9 @@ void AddNextExpansions(const Symbol &node_symbol, const Matching &matching, cons // We are not expanding from node1, so flip the expansion. DMG_ASSERT(expansion.node2 && symbol_table.at(*expansion.node2->identifier_) == node_symbol, "Expected node_symbol to be bound in node2"); - if (expansion.edge->type_ != EdgeAtom::Type::BREADTH_FIRST) { + if (expansion.edge->type_ != EdgeAtom::Type::BREADTH_FIRST && !expansion.edge->filter_lambda_.accumulated_path) { // BFS must *not* be flipped. Doing that changes the BFS results. + // When filter lambda uses accumulated path, path must not be flipped. std::swap(expansion.node1, expansion.node2); expansion.is_flipped = true; if (expansion.direction != EdgeAtom::Direction::BOTH) { diff --git a/tests/gql_behave/tests/memgraph_V1/features/match.feature b/tests/gql_behave/tests/memgraph_V1/features/match.feature index cf41c20f2..227ad9ad6 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/match.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/match.feature @@ -699,3 +699,75 @@ Feature: Match Then the result should be | date(n.time) | | 2021-10-05 | + + Scenario: Variable expand with filter by size of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:Person {id: 1})-[:KNOWS]->(:Person {id: 2})-[:KNOWS]->(:Person {id: 3})-[:KNOWS]->(:Person {id: 4}); + """ + When executing query: + """ + MATCH path = (:Person {id: 1})-[* (e, n, p | size(p) < 4)]->(:Person {id: 4}) RETURN path + """ + Then the result should be + | path | + | <(:Person{id:1})-[:KNOWS]->(:Person{id:2})-[:KNOWS]->(:Person{id:3})-[:KNOWS]->(:Person{id:4})> | + + Scenario: Variable expand with filter by last edge type of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:Person {id: 1})-[:KNOWS]->(:Person {id: 2})-[:KNOWS]->(:Person {id: 3})-[:KNOWS]->(:Person {id: 4}); + """ + When executing query: + """ + MATCH path = (:Person {id: 1})-[* (e, n, p | type(relationships(p)[-1]) = 'KNOWS')]->(:Person {id: 4}) RETURN path + """ + Then the result should be + | path | + | <(:Person{id:1})-[:KNOWS]->(:Person{id:2})-[:KNOWS]->(:Person{id:3})-[:KNOWS]->(:Person{id:4})> | + + Scenario: Variable expand with too restricted filter by size of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:Person {id: 1})-[:KNOWS]->(:Person {id: 2})-[:KNOWS]->(:Person {id: 3})-[:KNOWS]->(:Person {id: 4}); + """ + When executing query: + """ + MATCH path = (:Person {id: 1})-[* (e, n, p | size(p) < 3)]->(:Person {id: 4}) RETURN path + """ + Then the result should be empty + + Scenario: Variable expand with too restricted filter by last edge type of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:Person {id: 1})-[:KNOWS]->(:Person {id: 2})-[:KNOWS]->(:Person {id: 3})-[:KNOWS]->(:Person {id: 4}); + """ + When executing query: + """ + MATCH path = (:Person {id: 1})-[* (e, n, p | type(relationships(p)[-1]) = 'Invalid')]->(:Person {id: 4}) RETURN path + """ + Then the result should be empty + + Scenario: Test DFS variable expand with filter by edge type1 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[* (e, n, p | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1'))]->(:label3) RETURN path; + """ + Then the result should be: + | path | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | + + Scenario: Test DFS variable expand with filter by edge type2 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[* (e, n, p | NOT(type(e)='type2' AND type(last(relationships(p))) = 'type2'))]->(:label3) RETURN path; + """ + Then the result should be: + | path | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})> | diff --git a/tests/gql_behave/tests/memgraph_V1/features/memgraph_allshortest.feature b/tests/gql_behave/tests/memgraph_V1/features/memgraph_allshortest.feature index 73fb9e75b..29dc0a5ef 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/memgraph_allshortest.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/memgraph_allshortest.feature @@ -203,3 +203,103 @@ Feature: All Shortest Path Then the result should be: | total_cost | | 20.3 | + + Scenario: Test match AllShortest with accumulated path filtered by order of ids + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})-[:type1 {id: 3}]->(:label4 {id: 4}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*ALLSHORTEST (r, n | r.id) total_weight (e,n,p | e.id > 0 and (nodes(p)[-1]).id > (nodes(p)[-2]).id)]->(:label4) RETURN pth, total_weight; + """ + Then the result should be: + | pth | total_weight | + | <(:label1{id:1})-[:type1{id:1}]->(:label2{id:2})-[:type1{id:2}]->(:label3{id:3})-[:type1{id:3}]->(:label4{id:4})> | 6 | + + Scenario: Test match AllShortest with accumulated path filtered by edge type1 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | r.id) total_weight (e, n, p | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1'))]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 10 | + + Scenario: Test match AllShortest with accumulated path filtered by edge type2 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | r.id) total_weight (e, n, p | NOT(type(e)='type2' AND type(last(relationships(p))) = 'type2'))]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})> | 3 | + + Scenario: Test match AllShortest with accumulated path filtered by edge type1 and accumulated weight based on edge + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | r.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 10 | + + Scenario: Test match AllShortest with accumulated path filtered by edge type1 and accumulated weight based on edge too restricted + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | r.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w < 10)]->(:label3) RETURN path, total_weight; + """ + Then the result should be empty + + Scenario: Test match AllShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex is int + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | n.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 4 | + + Scenario: Test match allShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex and edge are ints + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | n.id + coalesce(r.id, 0)) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 14 | + + Scenario: Test match AllShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex and edge are doubles + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1.5}]->(:label2 {id: 2})-[:type1 {id: 2.1}]->(:label3 {id: 3})-[:type1 {id: 3.4}]->(:label4 {id: 4}); + """ + When executing query: + """ + MATCH path=(:label1)-[*ALLSHORTEST (r, n | n.id + coalesce(r.id, 0)) total_weight (e, n, p, w | w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type1 {id: 1.5}]->(:label2 {id: 2})-[:type1 {id: 2.1}]->(:label3 {id: 3})> | 9.6 | + + Scenario: Test match AllShortest with accumulated path filtered by order of ids and accumulated weight based on both vertex and edge is duration + Given an empty graph + And having executed: + """ + CREATE (:station {name: "A", arrival: localTime("08:00"), departure: localTime("08:15")})-[:ride {id: 1, duration: duration("PT1H5M")}]->(:station {name: "B", arrival: localtime("09:20"), departure: localTime("09:30")})-[:ride {id: 2, duration: duration("PT30M")}]->(:station {name: "C", arrival: localTime("10:00"), departure: localTime("10:20")}); + """ + When executing query: + """ + MATCH path=(:station {name:"A"})-[*ALLSHORTEST (r, v | v.departure - v.arrival + coalesce(r.duration, duration("PT0M"))) total_weight (r,n,p,w | (nodes(p)[-1]).name > (nodes(p)[-2]).name AND not(w is null))]->(:station {name:"C"}) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:station {arrival: 08:00:00.000000000, departure: 08:15:00.000000000, name: 'A'})-[:ride {duration: PT1H5M, id: 1}]->(:station {arrival: 09:20:00.000000000, departure: 09:30:00.000000000, name: 'B'})-[:ride {duration: PT30M, id: 2}]->(:station {arrival: 10:00:00.000000000, departure: 10:20:00.000000000, name: 'C'})> | PT2H20M | diff --git a/tests/gql_behave/tests/memgraph_V1/features/memgraph_bfs.feature b/tests/gql_behave/tests/memgraph_V1/features/memgraph_bfs.feature index d47566012..2736a6d71 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/memgraph_bfs.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/memgraph_bfs.feature @@ -121,3 +121,95 @@ Feature: Bfs Then the result should be: | p | | <(:Node {id: 2})-[:LINK {date: '2023-03'}]->(:Node {id: 3})> | + + Scenario: Test BFS variable expand with filter by last edge type of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | type(relationships(p)[-1]) = 'type1')]->(:label3) return pth; + """ + Then the result should be: + | pth | + | <(:label1{id:1})-[:type1{id:1}]->(:label2{id:2})-[:type1{id:2}]->(:label3{id:3})> | + + Scenario: Test BFS variable expand with restict filter by last edge type of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | type(relationships(p)[-1]) = 'type2')]->(:label2) return pth; + """ + Then the result should be empty + + Scenario: Test BFS variable expand with filter by size of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | size(p) < 3)]->(:label3) return pth; + """ + Then the result should be: + | pth | + | <(:label1{id:1})-[:type1{id:1}]->(:label2{id:2})-[:type1{id:2}]->(:label3{id:3})> | + + Scenario: Test BFS variable expand with restict filter by size of accumulated path + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | size(p) < 2)]->(:label3) return pth; + """ + Then the result should be empty + + Scenario: Test BFS variable expand with filter by order of ids in accumulated path when target vertex is indexed + Given graph "graph_index" + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | (nodes(p)[-1]).id > (nodes(p)[-2]).id)]->(:label4) return pth; + """ + Then the result should be: + | pth | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})-[:type1 {id: 3}]->(:label4 {id: 4})> | + + Scenario: Test BFS variable expand with filter by order of ids in accumulated path when target vertex is NOT indexed + Given graph "graph_index" + When executing query: + """ + MATCH pth=(:label1)-[*BFS (e,n,p | (nodes(p)[-1]).id > (nodes(p)[-2]).id)]->(:label3) return pth; + """ + Then the result should be: + | pth | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})> | + + Scenario: Test BFS variable expand with filter by edge type1 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*BFS (e, n, p | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1'))]->(:label3) RETURN path; + """ + Then the result should be: + | path | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | + + Scenario: Test BFS variable expand with filter by edge type2 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*BFS (e, n, p | NOT(type(e)='type2' AND type(last(relationships(p))) = 'type2'))]->(:label3) RETURN path; + """ + Then the result should be: + | path | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})> | diff --git a/tests/gql_behave/tests/memgraph_V1/features/memgraph_wshortest.feature b/tests/gql_behave/tests/memgraph_V1/features/memgraph_wshortest.feature index 1c98c2830..819bc94b3 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/memgraph_wshortest.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/memgraph_wshortest.feature @@ -155,3 +155,103 @@ Feature: Weighted Shortest Path MATCH (n {a:'0'})-[le *wShortest 10 (e, n | e.w ) w]->(m) RETURN m.a, size(le) as s, w """ Then an error should be raised + + Scenario: Test match wShortest with accumulated path filtered by order of ids + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})-[:type1 {id: 3}]->(:label4 {id: 4}); + """ + When executing query: + """ + MATCH pth=(:label1)-[*WSHORTEST (r, n | r.id) total_weight (e,n,p | e.id > 0 and (nodes(p)[-1]).id > (nodes(p)[-2]).id)]->(:label4) RETURN pth, total_weight; + """ + Then the result should be: + | pth | total_weight | + | <(:label1{id:1})-[:type1{id:1}]->(:label2{id:2})-[:type1{id:2}]->(:label3{id:3})-[:type1{id:3}]->(:label4{id:4})> | 6 | + + Scenario: Test match wShortest with accumulated path filtered by edge type1 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | r.id) total_weight (e, n, p | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1'))]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 10 | + + Scenario: Test match wShortest with accumulated path filtered by edge type2 + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | r.id) total_weight (e, n, p | NOT(type(e)='type2' AND type(last(relationships(p))) = 'type2'))]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type1 {id: 1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})> | 3 | + + Scenario: Test match wShortest with accumulated path filtered by edge type1 and accumulated weight based on edge + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | r.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 10 | + + Scenario: Test match wShortest with accumulated path filtered by edge type1 and accumulated weight based on edge too restricted + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | r.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w < 10)]->(:label3) RETURN path, total_weight; + """ + Then the result should be empty + + Scenario: Test match wShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex is int + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | n.id) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 4 | + + Scenario: Test match wShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex and edge are ints + Given graph "graph_edges" + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | n.id + coalesce(r.id, 0)) total_weight (e, n, p, w | NOT(type(e)='type1' AND type(last(relationships(p))) = 'type1') AND w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type2 {id: 10}]->(:label3 {id: 3})> | 14 | + + Scenario: Test match wShortest with accumulated path filtered by edge type1 and accumulated weight based on vertex and edge are doubles + Given an empty graph + And having executed: + """ + CREATE (:label1 {id: 1})-[:type1 {id:1.5}]->(:label2 {id: 2})-[:type1 {id: 2.1}]->(:label3 {id: 3})-[:type1 {id: 3.4}]->(:label4 {id: 4}); + """ + When executing query: + """ + MATCH path=(:label1)-[*WSHORTEST (r, n | n.id + coalesce(r.id, 0)) total_weight (e, n, p, w | w > 0)]->(:label3) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:label1 {id: 1})-[:type1 {id: 1.5}]->(:label2 {id: 2})-[:type1 {id: 2.1}]->(:label3 {id: 3})> | 9.6 | + + Scenario: Test match wShortest with accumulated path filtered by order of ids and accumulated weight based on both vertex and edge is duration + Given an empty graph + And having executed: + """ + CREATE (:station {name: "A", arrival: localTime("08:00"), departure: localTime("08:15")})-[:ride {id: 1, duration: duration("PT1H5M")}]->(:station {name: "B", arrival: localtime("09:20"), departure: localTime("09:30")})-[:ride {id: 2, duration: duration("PT30M")}]->(:station {name: "C", arrival: localTime("10:00"), departure: localTime("10:20")}); + """ + When executing query: + """ + MATCH path=(:station {name:"A"})-[*WSHORTEST (r, v | v.departure - v.arrival + coalesce(r.duration, duration("PT0M"))) total_weight (r,n,p,w | (nodes(p)[-1]).name > (nodes(p)[-2]).name AND not(w is null))]->(:station {name:"C"}) RETURN path, total_weight; + """ + Then the result should be: + | path | total_weight | + | <(:station {arrival: 08:00:00.000000000, departure: 08:15:00.000000000, name: 'A'})-[:ride {duration: PT1H5M, id: 1}]->(:station {arrival: 09:20:00.000000000, departure: 09:30:00.000000000, name: 'B'})-[:ride {duration: PT30M, id: 2}]->(:station {arrival: 10:00:00.000000000, departure: 10:20:00.000000000, name: 'C'})> | PT2H20M | diff --git a/tests/gql_behave/tests/memgraph_V1/graphs/graph_edges.cypher b/tests/gql_behave/tests/memgraph_V1/graphs/graph_edges.cypher new file mode 100644 index 000000000..06e7cdb5c --- /dev/null +++ b/tests/gql_behave/tests/memgraph_V1/graphs/graph_edges.cypher @@ -0,0 +1,2 @@ +CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})-[:type1 {id: 3}]->(:label4 {id: 4}); +MATCH (n :label1), (m :label3) CREATE (n)-[:type2 {id: 10}]->(m); diff --git a/tests/gql_behave/tests/memgraph_V1/graphs/graph_index.cypher b/tests/gql_behave/tests/memgraph_V1/graphs/graph_index.cypher new file mode 100644 index 000000000..ef30012d1 --- /dev/null +++ b/tests/gql_behave/tests/memgraph_V1/graphs/graph_index.cypher @@ -0,0 +1,2 @@ +CREATE INDEX ON :label4; +CREATE (:label1 {id: 1})-[:type1 {id:1}]->(:label2 {id: 2})-[:type1 {id: 2}]->(:label3 {id: 3})-[:type1 {id:3}]->(:label4 {id: 4}); diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index 54453de09..31fd95c6c 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -1925,6 +1925,41 @@ TEST_P(CypherMainVisitorTest, MatchBfsReturn) { ASSERT_TRUE(eq); } +TEST_P(CypherMainVisitorTest, MatchBfsFilterByPathReturn) { + auto &ast_generator = *GetParam(); + { + const auto *query = dynamic_cast( + ast_generator.ParseQuery("MATCH pth=(r:type1 {id: 1})<-[*BFS ..10 (e, n, p | startNode(relationships(e)[-1]) = " + "c:type2)]->(:type3 {id: 3}) RETURN pth;")); + ASSERT_TRUE(query); + ASSERT_TRUE(query->single_query_); + const auto *match = dynamic_cast(query->single_query_->clauses_[0]); + ASSERT_TRUE(match); + ASSERT_EQ(match->patterns_.size(), 1U); + ASSERT_EQ(match->patterns_[0]->atoms_.size(), 3U); + auto *bfs = dynamic_cast(match->patterns_[0]->atoms_[1]); + ASSERT_TRUE(bfs); + EXPECT_TRUE(bfs->IsVariable()); + EXPECT_EQ(bfs->filter_lambda_.inner_edge->name_, "e"); + EXPECT_TRUE(bfs->filter_lambda_.inner_edge->user_declared_); + EXPECT_EQ(bfs->filter_lambda_.inner_node->name_, "n"); + EXPECT_TRUE(bfs->filter_lambda_.inner_node->user_declared_); + EXPECT_EQ(bfs->filter_lambda_.accumulated_path->name_, "p"); + EXPECT_TRUE(bfs->filter_lambda_.accumulated_path->user_declared_); + EXPECT_EQ(bfs->filter_lambda_.accumulated_weight, nullptr); + } +} + +TEST_P(CypherMainVisitorTest, SemanticExceptionOnBfsFilterByWeight) { + auto &ast_generator = *GetParam(); + { + ASSERT_THROW(ast_generator.ParseQuery( + "MATCH pth=(:type1 {id: 1})<-[*BFS ..10 (e, n, p, w | startNode(relationships(e)[-1] AND w > 0) = " + "c:type2)]->(:type3 {id: 3}) RETURN pth;"), + SemanticException); + } +} + TEST_P(CypherMainVisitorTest, MatchVariableLambdaSymbols) { auto &ast_generator = *GetParam(); auto *query = dynamic_cast(ast_generator.ParseQuery("MATCH () -[*]- () RETURN *")); @@ -1981,6 +2016,57 @@ TEST_P(CypherMainVisitorTest, MatchWShortestReturn) { EXPECT_TRUE(shortest->total_weight_->user_declared_); } +TEST_P(CypherMainVisitorTest, MatchWShortestFilterByPathReturn) { + auto &ast_generator = *GetParam(); + { + const auto *query = dynamic_cast( + ast_generator.ParseQuery("MATCH pth=()-[r:type1 *wShortest 10 (we, wn | 42) total_weight " + "(e, n, p | startNode(relationships(e)[-1]) = c:type3)]->(:type2) RETURN pth")); + ASSERT_TRUE(query); + ASSERT_TRUE(query->single_query_); + const auto *match = dynamic_cast(query->single_query_->clauses_[0]); + ASSERT_TRUE(match); + ASSERT_EQ(match->patterns_.size(), 1U); + ASSERT_EQ(match->patterns_[0]->atoms_.size(), 3U); + auto *shortestPath = dynamic_cast(match->patterns_[0]->atoms_[1]); + ASSERT_TRUE(shortestPath); + EXPECT_TRUE(shortestPath->IsVariable()); + EXPECT_EQ(shortestPath->filter_lambda_.inner_edge->name_, "e"); + EXPECT_TRUE(shortestPath->filter_lambda_.inner_edge->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.inner_node->name_, "n"); + EXPECT_TRUE(shortestPath->filter_lambda_.inner_node->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.accumulated_path->name_, "p"); + EXPECT_TRUE(shortestPath->filter_lambda_.accumulated_path->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.accumulated_weight, nullptr); + } +} + +TEST_P(CypherMainVisitorTest, MatchWShortestFilterByPathWeightReturn) { + auto &ast_generator = *GetParam(); + { + const auto *query = dynamic_cast(ast_generator.ParseQuery( + "MATCH pth=()-[r:type1 *wShortest 10 (we, wn | 42) total_weight " + "(e, n, p, w | startNode(relationships(e)[-1]) = c:type3 AND w < 50)]->(:type2) RETURN pth")); + ASSERT_TRUE(query); + ASSERT_TRUE(query->single_query_); + const auto *match = dynamic_cast(query->single_query_->clauses_[0]); + ASSERT_TRUE(match); + ASSERT_EQ(match->patterns_.size(), 1U); + ASSERT_EQ(match->patterns_[0]->atoms_.size(), 3U); + auto *shortestPath = dynamic_cast(match->patterns_[0]->atoms_[1]); + ASSERT_TRUE(shortestPath); + EXPECT_TRUE(shortestPath->IsVariable()); + EXPECT_EQ(shortestPath->filter_lambda_.inner_edge->name_, "e"); + EXPECT_TRUE(shortestPath->filter_lambda_.inner_edge->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.inner_node->name_, "n"); + EXPECT_TRUE(shortestPath->filter_lambda_.inner_node->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.accumulated_path->name_, "p"); + EXPECT_TRUE(shortestPath->filter_lambda_.accumulated_path->user_declared_); + EXPECT_EQ(shortestPath->filter_lambda_.accumulated_weight->name_, "w"); + EXPECT_TRUE(shortestPath->filter_lambda_.accumulated_weight->user_declared_); + } +} + TEST_P(CypherMainVisitorTest, MatchWShortestNoFilterReturn) { auto &ast_generator = *GetParam(); auto *query = From d58a464141ee3265d85b6b3a03e0115316bc4cd0 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Sun, 3 Dec 2023 21:23:52 +0100 Subject: [PATCH 34/52] Remove filter profile info (#1481) --- src/query/plan/rewrite/index_lookup.hpp | 5 +++ src/query/plan/rewrite/join.hpp | 6 ++++ src/query/plan/rule_based_planner.hpp | 7 ++-- tests/e2e/CMakeLists.txt | 1 + tests/e2e/analyze_graph/optimize_indexes.py | 16 ++++----- tests/e2e/filter_info/CMakeLists.txt | 6 ++++ tests/e2e/filter_info/common.py | 23 ++++++++++++ tests/e2e/filter_info/filter_info.py | 39 +++++++++++++++++++++ tests/e2e/filter_info/workloads.yaml | 13 +++++++ tests/e2e/index_hints/index_hints.py | 19 +++++----- 10 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/filter_info/CMakeLists.txt create mode 100644 tests/e2e/filter_info/common.py create mode 100644 tests/e2e/filter_info/filter_info.py create mode 100644 tests/e2e/filter_info/workloads.yaml diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 7bb88f659..590bad5f4 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -106,6 +106,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { prev_ops_.pop_back(); ExpressionRemovalResult removal = RemoveExpressions(op.expression_, filter_exprs_for_removal_); op.expression_ = removal.trimmed_expression; + if (op.expression_) { + Filters leftover_filters; + leftover_filters.CollectFilterExpression(op.expression_, *symbol_table_); + op.all_filters_ = std::move(leftover_filters); + } // edge uniqueness filter comes always before filter in plan generation LogicalOperator *input = op.input().get(); diff --git a/src/query/plan/rewrite/join.hpp b/src/query/plan/rewrite/join.hpp index 65e32b3e8..e346ded45 100644 --- a/src/query/plan/rewrite/join.hpp +++ b/src/query/plan/rewrite/join.hpp @@ -59,6 +59,12 @@ class JoinRewriter final : public HierarchicalLogicalOperatorVisitor { ExpressionRemovalResult removal = RemoveExpressions(op.expression_, filter_exprs_for_removal_); op.expression_ = removal.trimmed_expression; + if (op.expression_) { + Filters leftover_filters; + leftover_filters.CollectFilterExpression(op.expression_, *symbol_table_); + op.all_filters_ = std::move(leftover_filters); + } + if (!op.expression_ || utils::Contains(filter_exprs_for_removal_, op.expression_)) { SetOnParent(op.input()); } diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index b58d0170b..a61658c90 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -897,13 +897,14 @@ class RuleBasedPlanner { std::unique_ptr GenFilters(std::unique_ptr last_op, const std::unordered_set &bound_symbols, Filters &filters, AstStorage &storage, const SymbolTable &symbol_table) { - auto all_filters = filters; auto pattern_filters = ExtractPatternFilters(filters, symbol_table, storage, bound_symbols); auto *filter_expr = impl::ExtractFilters(bound_symbols, filters, storage); if (filter_expr) { - last_op = - std::make_unique(std::move(last_op), std::move(pattern_filters), filter_expr, std::move(all_filters)); + Filters operator_filters; + operator_filters.CollectFilterExpression(filter_expr, symbol_table); + last_op = std::make_unique(std::move(last_op), std::move(pattern_filters), filter_expr, + std::move(operator_filters)); } return last_op; } diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 28fe94559..da96f6d4a 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -70,6 +70,7 @@ add_subdirectory(index_hints) add_subdirectory(query_modules) add_subdirectory(constraints) add_subdirectory(inspect_query) +add_subdirectory(filter_info) add_subdirectory(queries) add_subdirectory(garbage_collection) diff --git a/tests/e2e/analyze_graph/optimize_indexes.py b/tests/e2e/analyze_graph/optimize_indexes.py index 325993ca1..c0530a827 100644 --- a/tests/e2e/analyze_graph/optimize_indexes.py +++ b/tests/e2e/analyze_graph/optimize_indexes.py @@ -62,7 +62,7 @@ def test_analyze_graph_delete_statistics(delete_query, multi_db): # After deleting statistics, id2 should be chosen because it has less vertices expected_explain_after_delete_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id1}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id2}})",), (f" * Once",), ] @@ -96,7 +96,7 @@ def test_analyze_full_graph(analyze_query, multi_db): # Choose id2 before tha analysis because it has less vertices expected_explain_before_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id1}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id2}})",), (f" * Once",), ] @@ -117,7 +117,7 @@ def test_analyze_full_graph(analyze_query, multi_db): # After analyzing graph, id1 index should be chosen because it has smaller average group size expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id2}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id1}})",), (f" * Once",), ] @@ -152,7 +152,7 @@ def test_cardinality_different_avg_group_size_uniform_dist(multi_db): assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 20, 5, 0, 0) expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id2}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id1}})",), (f" * Once",), ] @@ -183,7 +183,7 @@ def test_cardinality_same_avg_group_size_uniform_dist_diff_vertex_count(multi_db assert analyze_graph_results[1 - first_index] == ("Label", "id2", 50, 50, 1, 0, 0) expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id1}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id2}})",), (f" * Once",), ] @@ -214,7 +214,7 @@ def test_large_diff_in_num_vertices_v1(multi_db): assert analyze_graph_results[1 - first_index] == ("Label", "id2", 99, 1, 99, 0, 0) expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id1}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id2}})",), (f" * Once",), ] @@ -245,7 +245,7 @@ def test_large_diff_in_num_vertices_v2(multi_db): assert analyze_graph_results[1 - first_index] == ("Label", "id2", 1000, 1000, 1, 0, 0) expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id2}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id1}})",), (f" * Once",), ] @@ -286,7 +286,7 @@ def test_same_avg_group_size_diff_distribution(multi_db): assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 5, 20, 0, 0) expected_explain_after_analysis = [ (f" * Produce {{n}}",), - (f" * Filter (n :Label), {{n.id1}}, {{n.id2}}",), + (f" * Filter {{n.id1}}",), (f" * ScanAllByLabelPropertyValue (n :Label {{id2}})",), (f" * Once",), ] diff --git a/tests/e2e/filter_info/CMakeLists.txt b/tests/e2e/filter_info/CMakeLists.txt new file mode 100644 index 000000000..401ec46a6 --- /dev/null +++ b/tests/e2e/filter_info/CMakeLists.txt @@ -0,0 +1,6 @@ +function(copy_filter_info_e2e_python_files FILE_NAME) + copy_e2e_python_files(filter_info ${FILE_NAME}) +endfunction() + +copy_filter_info_e2e_python_files(common.py) +copy_filter_info_e2e_python_files(filter_info.py) diff --git a/tests/e2e/filter_info/common.py b/tests/e2e/filter_info/common.py new file mode 100644 index 000000000..1eb81a94b --- /dev/null +++ b/tests/e2e/filter_info/common.py @@ -0,0 +1,23 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import pytest +from gqlalchemy import Memgraph + + +@pytest.fixture +def memgraph(**kwargs) -> Memgraph: + memgraph = Memgraph() + + yield memgraph + + memgraph.drop_database() + memgraph.drop_indexes() diff --git a/tests/e2e/filter_info/filter_info.py b/tests/e2e/filter_info/filter_info.py new file mode 100644 index 000000000..53bc737ea --- /dev/null +++ b/tests/e2e/filter_info/filter_info.py @@ -0,0 +1,39 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys + +import pytest +from common import memgraph + + +def test_label_index_hint(memgraph): + memgraph.execute("CREATE (n:Label1:Label2 {prop: 1});") + memgraph.execute("CREATE INDEX ON :Label1;") + + # TODO: Fix this test since it should only filter on :Label2 and prop + expected_explain = [ + " * Produce {n}", + " * Filter (n :Label1:Label2), {n.prop}", + " * ScanAllByLabel (n :Label1)", + " * Once", + ] + + actual_explain = [ + row["QUERY PLAN"] + for row in memgraph.execute_and_fetch("EXPLAIN MATCH (n:Label1:Label2) WHERE n.prop = 1 return n;") + ] + + assert expected_explain == actual_explain + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/filter_info/workloads.yaml b/tests/e2e/filter_info/workloads.yaml new file mode 100644 index 000000000..69c98f741 --- /dev/null +++ b/tests/e2e/filter_info/workloads.yaml @@ -0,0 +1,13 @@ +filter_info_cluster: &filter_info_cluster + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "filter_info.log" + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Filter info information" + binary: "tests/e2e/pytest_runner.sh" + args: ["filter_info/filter_info.py"] + <<: *filter_info_cluster diff --git a/tests/e2e/index_hints/index_hints.py b/tests/e2e/index_hints/index_hints.py index 85b63a84b..70d3ce6b6 100644 --- a/tests/e2e/index_hints/index_hints.py +++ b/tests/e2e/index_hints/index_hints.py @@ -162,12 +162,13 @@ def test_label_property_index_hint(memgraph): expected_explain_no_hint = [ " * Produce {n}", - " * Filter (n :Label), {n.id1}, {n.id2}", + " * Filter {n.id1}", " * ScanAllByLabelPropertyValue (n :Label {id2})", " * Once", ] expected_explain_with_hint = [ - row.replace("(n :Label {id2})", "(n :Label {id1})") for row in expected_explain_no_hint + row.replace("(n :Label {id2})", "(n :Label {id1})").replace(" * Filter {n.id1}", " * Filter {n.id2}") + for row in expected_explain_no_hint ] explain_no_hint = [ @@ -192,7 +193,7 @@ def test_label_property_index_hint_alternative_orderings(memgraph): expected_explain_with_hint = [ " * Produce {n}", - " * Filter (n :Label), {n.id1}, {n.id2}", + " * Filter {n.id2}", " * ScanAllByLabelPropertyValue (n :Label {id1})", " * Once", ] @@ -221,7 +222,7 @@ def test_multiple_label_property_index_hints(memgraph): expected_explain_with_hint = [ " * Produce {n}", - " * Filter (n :Label), {n.id1}, {n.id2}", + " * Filter {n.id2}", " * ScanAllByLabelPropertyValue (n :Label {id1})", " * Once", ] @@ -251,7 +252,7 @@ def test_multiple_applicable_label_property_index_hints(memgraph): expected_explain_with_hint = [ " * Produce {n}", - " * Filter (n :Label), {n.id1}, {n.id2}", + " * Filter {n.id2}", " * ScanAllByLabelPropertyValue (n :Label {id1})", " * Once", ] @@ -275,12 +276,13 @@ def test_multiple_applicable_label_property_index_hints_alternative_orderings(me expected_explain_with_hint_1 = [ " * Produce {n}", - " * Filter (n :Label), {n.id1}, {n.id2}", + " * Filter {n.id2}", " * ScanAllByLabelPropertyValue (n :Label {id1})", " * Once", ] expected_explain_with_hint_2 = [ - row.replace("(n :Label {id1})", "(n :Label {id2})") for row in expected_explain_with_hint_1 + row.replace("(n :Label {id1})", "(n :Label {id2})").replace(" * Filter {n.id2}", " * Filter {n.id1}") + for row in expected_explain_with_hint_1 ] explain_with_hint_ordering_1a = [ @@ -407,6 +409,7 @@ def test_multiple_match_query(memgraph): memgraph.execute("CREATE INDEX ON :Label2;") memgraph.execute("CREATE INDEX ON :Label3;") + # TODO: Fix this test since it has the filtering info wrong (filtering by label that's already indexed) expected_explain_with_hint = [ " * Produce {n, m}", " * Cartesian {m : n}", @@ -414,7 +417,7 @@ def test_multiple_match_query(memgraph): " | * Filter (n :Label1:Label2), {n.id}", " | * ScanAllByLabel (n :Label1)", " | * Once", - " * Filter (m :Label2:Label3), (n :Label1:Label2), {n.id}", + " * Filter (m :Label2:Label3)", " * ScanAllByLabel (m :Label2)", " * Once", ] From 46bfeb002347424af7b8f2c0fa7e3767e0a93f1d Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Sun, 3 Dec 2023 22:28:26 +0100 Subject: [PATCH 35/52] Fix counting when no matched nodes by property (#1518) --- src/query/interpret/eval.hpp | 2 + src/query/plan/operator.cpp | 1 + .../memgraph_V1/features/aggregations.feature | 62 ++++++++++++++++--- .../features/aggregations.feature | 62 ++++++++++++++++--- 4 files changed, 113 insertions(+), 14 deletions(-) diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index e09ddcc97..940aec710 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -188,6 +188,8 @@ class ExpressionEvaluator : public ExpressionVisitor { utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; } + void ResetPropertyLookupCache() { property_lookup_cache_.clear(); } + TypedValue Visit(NamedExpression &named_expression) override { const auto &symbol = symbol_table_->at(named_expression); auto value = named_expression.expression_->Accept(*this); diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 24ce66a69..79a4f7d75 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -3676,6 +3676,7 @@ class AggregateCursor : public Cursor { void ProcessOne(const Frame &frame, ExpressionEvaluator *evaluator) { // Preallocated group_by, since most of the time the aggregation key won't be unique reused_group_by_.clear(); + evaluator->ResetPropertyLookupCache(); for (Expression *expression : self_.group_by_) { reused_group_by_.emplace_back(expression->Accept(*evaluator)); diff --git a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature index 80b0ca69a..11297f39e 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature @@ -410,19 +410,67 @@ Feature: Aggregations CREATE (s:Subnet {ip: "192.168.0.1"}) """ When executing query: - """ - MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id - """ + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id + """ Then the result should be empty - Scenario: Empty count aggregation: + Scenario: Empty count aggregation Given an empty graph And having executed """ CREATE (s:Subnet {ip: "192.168.0.1"}) """ When executing query: - """ - MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id - """ + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id + """ Then the result should be empty + + Scenario: Collect nodes properties into a map: + Given an empty graph + And having executed + """ + CREATE (t:Tag {short_code: "TST", description: "SYSTEM_TAG"}), (t2:Tag {short_code: "PRD", description: "SYSTEM_TAG"}), + (t3:Tag {short_code: "STG", description: "SYSTEM_TAG"}), (device {name: "name1"}), (device)-[a1:ASSOCIATED]->(t), + (device)-[a2:ASSOCIATED]->(t2), (device)-[a3:ASSOCIATED]->(t3); + """ + When executing query: + """ + MATCH (d {name: "name1"})-[t:ASSOCIATED]-(tag:Tag) RETURN collect({short_code: tag.short_code, description: tag.description}) as tags; + """ + Then the result should be: + | tags | + | [{description: 'SYSTEM_TAG', short_code: 'TST'}, {description: 'SYSTEM_TAG', short_code: 'PRD'}, {description: 'SYSTEM_TAG', short_code: 'STG'}] | + + Scenario: Count directly without WITH clause 01 + Given an empty graph + And having executed + """ + CREATE (:Node {prop1: 1, prop2: 2, prop3: 3}), (:Node {prop1: 10, prop2: 11, prop3: 12}), (:Node {prop1: 20, prop2: 21, prop3: 22}) + """ + When executing query: + """ + MATCH (n) RETURN n.prop1, n.prop2, n.prop3, count(*) AS cnt + """ + Then the result should be: + | n.prop1 | n.prop2 | n.prop3 | cnt | + | 20 | 21 | 22 | 1 | + | 10 | 11 | 12 | 1 | + | 1 | 2 | 3 | 1 | + + Scenario: Count directly without WITH clause 02 + Given an empty graph + And having executed + """ + CREATE (:Node {prop1: 1, prop2: 2, prop3: 3}), (:Node {prop1: 10, prop2: 11, prop3: 12}), (:Node {prop1: 20, prop2: 21, prop3: 22}) + """ + When executing query: + """ + MATCH (n) WITH n.prop1 AS prop1, n.prop2 AS prop2, n.prop3 AS prop3 RETURN prop1, prop2, prop3, count(*) AS cnt; + """ + Then the result should be: + | prop1 | prop2 | prop3 | cnt | + | 20 | 21 | 22 | 1 | + | 10 | 11 | 12 | 1 | + | 1 | 2 | 3 | 1 | diff --git a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature index 80b0ca69a..11297f39e 100644 --- a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature @@ -410,19 +410,67 @@ Feature: Aggregations CREATE (s:Subnet {ip: "192.168.0.1"}) """ When executing query: - """ - MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id - """ + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, collect(subnet.ip) as ips RETURN id(subnet) as id + """ Then the result should be empty - Scenario: Empty count aggregation: + Scenario: Empty count aggregation Given an empty graph And having executed """ CREATE (s:Subnet {ip: "192.168.0.1"}) """ When executing query: - """ - MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id - """ + """ + MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id + """ Then the result should be empty + + Scenario: Collect nodes properties into a map: + Given an empty graph + And having executed + """ + CREATE (t:Tag {short_code: "TST", description: "SYSTEM_TAG"}), (t2:Tag {short_code: "PRD", description: "SYSTEM_TAG"}), + (t3:Tag {short_code: "STG", description: "SYSTEM_TAG"}), (device {name: "name1"}), (device)-[a1:ASSOCIATED]->(t), + (device)-[a2:ASSOCIATED]->(t2), (device)-[a3:ASSOCIATED]->(t3); + """ + When executing query: + """ + MATCH (d {name: "name1"})-[t:ASSOCIATED]-(tag:Tag) RETURN collect({short_code: tag.short_code, description: tag.description}) as tags; + """ + Then the result should be: + | tags | + | [{description: 'SYSTEM_TAG', short_code: 'TST'}, {description: 'SYSTEM_TAG', short_code: 'PRD'}, {description: 'SYSTEM_TAG', short_code: 'STG'}] | + + Scenario: Count directly without WITH clause 01 + Given an empty graph + And having executed + """ + CREATE (:Node {prop1: 1, prop2: 2, prop3: 3}), (:Node {prop1: 10, prop2: 11, prop3: 12}), (:Node {prop1: 20, prop2: 21, prop3: 22}) + """ + When executing query: + """ + MATCH (n) RETURN n.prop1, n.prop2, n.prop3, count(*) AS cnt + """ + Then the result should be: + | n.prop1 | n.prop2 | n.prop3 | cnt | + | 20 | 21 | 22 | 1 | + | 10 | 11 | 12 | 1 | + | 1 | 2 | 3 | 1 | + + Scenario: Count directly without WITH clause 02 + Given an empty graph + And having executed + """ + CREATE (:Node {prop1: 1, prop2: 2, prop3: 3}), (:Node {prop1: 10, prop2: 11, prop3: 12}), (:Node {prop1: 20, prop2: 21, prop3: 22}) + """ + When executing query: + """ + MATCH (n) WITH n.prop1 AS prop1, n.prop2 AS prop2, n.prop3 AS prop3 RETURN prop1, prop2, prop3, count(*) AS cnt; + """ + Then the result should be: + | prop1 | prop2 | prop3 | cnt | + | 20 | 21 | 22 | 1 | + | 10 | 11 | 12 | 1 | + | 1 | 2 | 3 | 1 | From 0fb3ae2d5671f366f60dd1b6973b8f3143ae3107 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Mon, 4 Dec 2023 00:01:29 +0100 Subject: [PATCH 36/52] Fix three match cartesian sequential scanning (#1555) --- src/query/plan/rule_based_planner.hpp | 12 ++++-- tests/e2e/CMakeLists.txt | 1 + tests/e2e/query_planning/CMakeLists.txt | 6 +++ tests/e2e/query_planning/common.py | 24 +++++++++++ .../query_planning_cartesian.py | 42 +++++++++++++++++++ tests/e2e/query_planning/workloads.yaml | 14 +++++++ 6 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/query_planning/CMakeLists.txt create mode 100644 tests/e2e/query_planning/common.py create mode 100644 tests/e2e/query_planning/query_planning_cartesian.py create mode 100644 tests/e2e/query_planning/workloads.yaml diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index a61658c90..074bd1c88 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -511,10 +511,6 @@ class RuleBasedPlanner { std::set visited_expansion_groups; - last_op = - GenerateExpansionOnAlreadySeenSymbols(std::move(last_op), matching, visited_expansion_groups, symbol_table, - storage, bound_symbols, new_symbols, named_paths, filters, view); - // We want to create separate branches of scan operators for each expansion group group of patterns // Whenever there are 2 scan branches, they will be joined with a Cartesian operator @@ -528,6 +524,14 @@ class RuleBasedPlanner { continue; } + last_op = + GenerateExpansionOnAlreadySeenSymbols(std::move(last_op), matching, visited_expansion_groups, symbol_table, + storage, bound_symbols, new_symbols, named_paths, filters, view); + + if (visited_expansion_groups.contains(expansion.expansion_group_id)) { + continue; + } + std::unique_ptr starting_expansion_operator = nullptr; if (!initial_expansion_done) { starting_expansion_operator = std::move(last_op); diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index da96f6d4a..594c9acb0 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -73,6 +73,7 @@ add_subdirectory(inspect_query) add_subdirectory(filter_info) add_subdirectory(queries) add_subdirectory(garbage_collection) +add_subdirectory(query_planning) copy_e2e_python_files(pytest_runner pytest_runner.sh "") copy_e2e_python_files(x x.sh "") diff --git a/tests/e2e/query_planning/CMakeLists.txt b/tests/e2e/query_planning/CMakeLists.txt new file mode 100644 index 000000000..9c6d39bf9 --- /dev/null +++ b/tests/e2e/query_planning/CMakeLists.txt @@ -0,0 +1,6 @@ +function(copy_query_planning_e2e_python_files FILE_NAME) + copy_e2e_python_files(query_planning ${FILE_NAME}) +endfunction() + +copy_query_planning_e2e_python_files(common.py) +copy_query_planning_e2e_python_files(query_planning_cartesian.py) diff --git a/tests/e2e/query_planning/common.py b/tests/e2e/query_planning/common.py new file mode 100644 index 000000000..6ad52539b --- /dev/null +++ b/tests/e2e/query_planning/common.py @@ -0,0 +1,24 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import pytest +from gqlalchemy import Memgraph + + +@pytest.fixture +def memgraph(**kwargs) -> Memgraph: + memgraph = Memgraph() + + yield memgraph + + memgraph.drop_indexes() + memgraph.ensure_constraints([]) + memgraph.drop_database() diff --git a/tests/e2e/query_planning/query_planning_cartesian.py b/tests/e2e/query_planning/query_planning_cartesian.py new file mode 100644 index 000000000..11bc3f628 --- /dev/null +++ b/tests/e2e/query_planning/query_planning_cartesian.py @@ -0,0 +1,42 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import sys + +import pytest +from common import memgraph + +QUERY_PLAN = "QUERY PLAN" + + +def test_indexed_join_with_indices(memgraph): + memgraph.execute("CREATE INDEX ON :Node(id);") + + expected_explain = [ + f" * Produce {{a, b, r}}", + f" * Filter (a :Node), {{a.id}}", + f" * Expand (b)-[r:EDGE]-(a)", + f" * ScanAllByLabelPropertyValue (b :Node {{id}})", + f" * Once", + ] + + results = list( + memgraph.execute_and_fetch( + "EXPLAIN MATCH (a:Node {id: 1}) MATCH (b:Node {id: 2}) MATCH (a)-[r:EDGE]-(b) return a,b,r;" + ) + ) + actual_explain = [x[QUERY_PLAN] for x in results] + + assert expected_explain == actual_explain + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/query_planning/workloads.yaml b/tests/e2e/query_planning/workloads.yaml new file mode 100644 index 000000000..3fd5fb478 --- /dev/null +++ b/tests/e2e/query_planning/workloads.yaml @@ -0,0 +1,14 @@ +queries_cluster: &queries_cluster + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "query_planning.log" + setup_queries: [] + validation_queries: [] + + +workloads: + - name: "Query planning cartesian" + binary: "tests/e2e/pytest_runner.sh" + args: ["query_planning/query_planning_cartesian.py"] + <<: *queries_cluster From 953a8f53402e47f10d0b10b1547f3c4179feb150 Mon Sep 17 00:00:00 2001 From: Aidar Samerkhanov Date: Mon, 4 Dec 2023 10:32:59 +0300 Subject: [PATCH 37/52] Add handling of deleted return values for query procedures and functions ran in analytical mode (#1395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ante PuÅ¡ić --- include/_mgp.hpp | 16 +- include/mg_procedure.h | 21 ++ include/mgp.hpp | 29 ++ src/dbms/database.hpp | 2 +- src/query/db_accessor.cpp | 3 + src/query/db_accessor.hpp | 6 +- src/query/interpret/eval.hpp | 17 +- src/query/plan/operator.cpp | 15 +- src/query/procedure/mg_procedure_impl.cpp | 86 +++++- src/query/procedure/mg_procedure_impl.hpp | 16 +- src/query/procedure/py_module.cpp | 127 ++++++-- src/query/stream/streams.cpp | 2 +- src/query/typed_value.cpp | 32 ++ src/query/typed_value.hpp | 2 + src/storage/v2/edge_accessor.cpp | 7 + src/storage/v2/edge_accessor.hpp | 2 + src/storage/v2/storage.cpp | 4 +- src/storage/v2/storage.hpp | 4 +- src/storage/v2/storage_mode.cpp | 4 + src/storage/v2/storage_mode.hpp | 2 + tests/e2e/CMakeLists.txt | 1 + .../CMakeLists.txt | 8 + .../e2e/query_modules_storage_modes/common.py | 37 +++ .../query_modules/CMakeLists.txt | 4 + .../query_modules/c_api.cpp | 70 +++++ .../query_modules/cpp_api.cpp | 86 ++++++ .../query_modules/python_api.py | 55 ++++ .../test_query_modules_storage_modes.py | 283 ++++++++++++++++++ .../workloads.yaml | 14 + tests/gql_behave/continuous_integration | 2 +- tests/unit/cpp_api.cpp | 4 +- tests/unit/query_procedure_mgp_type.cpp | 6 +- tests/unit/query_procedure_py_module.cpp | 6 +- tests/unit/query_procedures_mgp_graph.cpp | 3 +- 34 files changed, 919 insertions(+), 57 deletions(-) create mode 100644 tests/e2e/query_modules_storage_modes/CMakeLists.txt create mode 100644 tests/e2e/query_modules_storage_modes/common.py create mode 100644 tests/e2e/query_modules_storage_modes/query_modules/CMakeLists.txt create mode 100644 tests/e2e/query_modules_storage_modes/query_modules/c_api.cpp create mode 100644 tests/e2e/query_modules_storage_modes/query_modules/cpp_api.cpp create mode 100644 tests/e2e/query_modules_storage_modes/query_modules/python_api.py create mode 100644 tests/e2e/query_modules_storage_modes/test_query_modules_storage_modes.py create mode 100644 tests/e2e/query_modules_storage_modes/workloads.yaml diff --git a/include/_mgp.hpp b/include/_mgp.hpp index 58685b440..4f6797739 100644 --- a/include/_mgp.hpp +++ b/include/_mgp.hpp @@ -234,8 +234,6 @@ inline mgp_type *type_duration() { return MgInvoke(mgp_type_duration inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke(mgp_type_nullable, type); } -// mgp_graph - inline bool create_label_index(mgp_graph *graph, const char *label) { return MgInvoke(mgp_create_label_index, graph, label); } @@ -284,6 +282,10 @@ inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memor return MgInvoke(mgp_list_all_unique_constraints, graph, memory); } +// mgp_graph + +inline bool graph_is_transactional(mgp_graph *graph) { return MgInvoke(mgp_graph_is_transactional, graph); } + inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke(mgp_graph_is_mutable, graph); } inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) { @@ -376,6 +378,8 @@ inline mgp_list *list_copy(mgp_list *list, mgp_memory *memory) { inline void list_destroy(mgp_list *list) { mgp_list_destroy(list); } +inline bool list_contains_deleted(mgp_list *list) { return MgInvoke(mgp_list_contains_deleted, list); } + inline void list_append(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append, list, val); } inline void list_append_extend(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append_extend, list, val); } @@ -394,6 +398,8 @@ inline mgp_map *map_copy(mgp_map *map, mgp_memory *memory) { return MgInvoke(mgp_map_contains_deleted, map); } + inline void map_insert(mgp_map *map, const char *key, mgp_value *value) { MgInvokeVoid(mgp_map_insert, map, key, value); } @@ -442,6 +448,8 @@ inline mgp_vertex *vertex_copy(mgp_vertex *v, mgp_memory *memory) { inline void vertex_destroy(mgp_vertex *v) { mgp_vertex_destroy(v); } +inline bool vertex_is_deleted(mgp_vertex *v) { return MgInvoke(mgp_vertex_is_deleted, v); } + inline bool vertex_equal(mgp_vertex *v1, mgp_vertex *v2) { return MgInvoke(mgp_vertex_equal, v1, v2); } inline size_t vertex_labels_count(mgp_vertex *v) { return MgInvoke(mgp_vertex_labels_count, v); } @@ -494,6 +502,8 @@ inline mgp_edge *edge_copy(mgp_edge *e, mgp_memory *memory) { return MgInvoke(mgp_edge_is_deleted, e); } + inline bool edge_equal(mgp_edge *e1, mgp_edge *e2) { return MgInvoke(mgp_edge_equal, e1, e2); } inline mgp_edge_type edge_get_type(mgp_edge *e) { return MgInvoke(mgp_edge_get_type, e); } @@ -530,6 +540,8 @@ inline mgp_path *path_copy(mgp_path *path, mgp_memory *memory) { inline void path_destroy(mgp_path *path) { mgp_path_destroy(path); } +inline bool path_contains_deleted(mgp_path *path) { return MgInvoke(mgp_path_contains_deleted, path); } + inline void path_expand(mgp_path *path, mgp_edge *edge) { MgInvokeVoid(mgp_path_expand, path, edge); } inline void path_pop(mgp_path *path) { MgInvokeVoid(mgp_path_pop, path); } diff --git a/include/mg_procedure.h b/include/mg_procedure.h index 857c5f4dd..93ef241d8 100644 --- a/include/mg_procedure.h +++ b/include/mg_procedure.h @@ -429,6 +429,9 @@ enum mgp_error mgp_list_copy(struct mgp_list *list, struct mgp_memory *memory, s /// Free the memory used by the given mgp_list and contained elements. void mgp_list_destroy(struct mgp_list *list); +/// Return whether the given mgp_list contains any deleted values. +enum mgp_error mgp_list_contains_deleted(struct mgp_list *list, int *result); + /// Append a copy of mgp_value to mgp_list if capacity allows. /// The list copies the given value and therefore does not take ownership of the /// original value. You still need to call mgp_value_destroy to free the @@ -469,6 +472,9 @@ enum mgp_error mgp_map_copy(struct mgp_map *map, struct mgp_memory *memory, stru /// Free the memory used by the given mgp_map and contained items. void mgp_map_destroy(struct mgp_map *map); +/// Return whether the given mgp_map contains any deleted values. +enum mgp_error mgp_map_contains_deleted(struct mgp_map *map, int *result); + /// Insert a new mapping from a NULL terminated character string to a value. /// If a mapping with the same key already exists, it is *not* replaced. /// In case of insertion, both the string and the value are copied into the map. @@ -552,6 +558,9 @@ enum mgp_error mgp_path_copy(struct mgp_path *path, struct mgp_memory *memory, s /// Free the memory used by the given mgp_path and contained vertices and edges. void mgp_path_destroy(struct mgp_path *path); +/// Return whether the given mgp_path contains any deleted values. +enum mgp_error mgp_path_contains_deleted(struct mgp_path *path, int *result); + /// Append an edge continuing from the last vertex on the path. /// The edge is copied into the path. Therefore, the path does not take /// ownership of the original edge, so you still need to free the edge memory @@ -725,6 +734,9 @@ enum mgp_error mgp_vertex_copy(struct mgp_vertex *v, struct mgp_memory *memory, /// Free the memory used by a mgp_vertex. void mgp_vertex_destroy(struct mgp_vertex *v); +/// Return whether the given mgp_vertex is deleted. +enum mgp_error mgp_vertex_is_deleted(struct mgp_vertex *v, int *result); + /// Result is non-zero if given vertices are equal, otherwise 0. enum mgp_error mgp_vertex_equal(struct mgp_vertex *v1, struct mgp_vertex *v2, int *result); @@ -819,6 +831,9 @@ enum mgp_error mgp_edge_copy(struct mgp_edge *e, struct mgp_memory *memory, stru /// Free the memory used by a mgp_edge. void mgp_edge_destroy(struct mgp_edge *e); +/// Return whether the given mgp_edge is deleted. +enum mgp_error mgp_edge_is_deleted(struct mgp_edge *e, int *result); + /// Result is non-zero if given edges are equal, otherwise 0. enum mgp_error mgp_edge_equal(struct mgp_edge *e1, struct mgp_edge *e2, int *result); @@ -941,6 +956,12 @@ enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct m /// Current implementation always returns without errors. enum mgp_error mgp_graph_is_mutable(struct mgp_graph *graph, int *result); +/// Result is non-zero if the graph is in transactional storage mode. +/// If a graph is not in transactional mode (i.e. analytical mode), then vertices and edges can be missing +/// because changes from other transactions are visible. +/// Current implementation always returns without errors. +enum mgp_error mgp_graph_is_transactional(struct mgp_graph *graph, int *result); + /// Add a new vertex to the graph. /// Resulting vertex must be freed using mgp_vertex_destroy. /// Return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT if `graph` is immutable. diff --git a/include/mgp.hpp b/include/mgp.hpp index 6296d2e5c..3f7ed591e 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -246,6 +246,8 @@ class Graph { /// @brief Returns whether the graph is mutable. bool IsMutable() const; + /// @brief Returns whether the graph is in a transactional storage mode. + bool IsTransactional() const; /// @brief Creates a node and adds it to the graph. Node CreateNode(); /// @brief Deletes a node from the graph. @@ -512,6 +514,9 @@ class List { ~List(); + /// @brief Returns wheter the list contains any deleted values. + bool ContainsDeleted() const; + /// @brief Returns the size of the list. size_t Size() const; /// @brief Returns whether the list is empty. @@ -618,6 +623,9 @@ class Map { ~Map(); + /// @brief Returns wheter the map contains any deleted values. + bool ContainsDeleted() const; + /// @brief Returns the size of the map. size_t Size() const; @@ -730,6 +738,9 @@ class Node { ~Node(); + /// @brief Returns wheter the node has been deleted. + bool IsDeleted() const; + /// @brief Returns the node’s ID. mgp::Id Id() const; @@ -811,6 +822,9 @@ class Relationship { ~Relationship(); + /// @brief Returns wheter the relationship has been deleted. + bool IsDeleted() const; + /// @brief Returns the relationship’s ID. mgp::Id Id() const; @@ -876,6 +890,9 @@ class Path { ~Path(); + /// @brief Returns wheter the path contains any deleted values. + bool ContainsDeleted() const; + /// Returns the path length (number of relationships). size_t Length() const; @@ -1995,6 +2012,8 @@ inline bool Graph::ContainsRelationship(const Relationship &relationship) const inline bool Graph::IsMutable() const { return mgp::graph_is_mutable(graph_); } +inline bool Graph::IsTransactional() const { return mgp::graph_is_transactional(graph_); } + inline Node Graph::CreateNode() { auto *vertex = mgp::MemHandlerCallback(graph_create_vertex, graph_); auto node = Node(vertex); @@ -2442,6 +2461,8 @@ inline List::~List() { } } +inline bool List::ContainsDeleted() const { return mgp::list_contains_deleted(ptr_); } + inline size_t List::Size() const { return mgp::list_size(ptr_); } inline bool List::Empty() const { return Size() == 0; } @@ -2568,6 +2589,8 @@ inline Map::~Map() { } } +inline bool Map::ContainsDeleted() const { return mgp::map_contains_deleted(ptr_); } + inline size_t Map::Size() const { return mgp::map_size(ptr_); } inline bool Map::Empty() const { return Size() == 0; } @@ -2733,6 +2756,8 @@ inline Node::~Node() { } } +inline bool Node::IsDeleted() const { return mgp::vertex_is_deleted(ptr_); } + inline mgp::Id Node::Id() const { return Id::FromInt(mgp::vertex_get_id(ptr_).as_int); } inline mgp::Labels Node::Labels() const { return mgp::Labels(ptr_); } @@ -2884,6 +2909,8 @@ inline Relationship::~Relationship() { } } +inline bool Relationship::IsDeleted() const { return mgp::edge_is_deleted(ptr_); } + inline mgp::Id Relationship::Id() const { return Id::FromInt(mgp::edge_get_id(ptr_).as_int); } inline std::string_view Relationship::Type() const { return mgp::edge_get_type(ptr_).name; } @@ -2989,6 +3016,8 @@ inline Path::~Path() { } } +inline bool Path::ContainsDeleted() const { return mgp::path_contains_deleted(ptr_); } + inline size_t Path::Length() const { return mgp::path_size(ptr_); } inline Node Path::GetNodeAt(size_t index) const { diff --git a/src/dbms/database.hpp b/src/dbms/database.hpp index 416ff76bc..878fe7672 100644 --- a/src/dbms/database.hpp +++ b/src/dbms/database.hpp @@ -95,7 +95,7 @@ class Database { * * @return storage::StorageMode */ - storage::StorageMode GetStorageMode() const { return storage_->GetStorageMode(); } + storage::StorageMode GetStorageMode() const noexcept { return storage_->GetStorageMode(); } /** * @brief Get the storage info diff --git a/src/query/db_accessor.cpp b/src/query/db_accessor.cpp index df3fb808a..16ad399a0 100644 --- a/src/query/db_accessor.cpp +++ b/src/query/db_accessor.cpp @@ -15,6 +15,7 @@ #include #include +#include "storage/v2/storage_mode.hpp" #include "utils/pmr/unordered_set.hpp" namespace memgraph::query { @@ -139,6 +140,8 @@ std::optional SubgraphDbAccessor::FindVertex(storage::Gid gid, s query::Graph *SubgraphDbAccessor::getGraph() { return graph_; } +storage::StorageMode SubgraphDbAccessor::GetStorageMode() const noexcept { return db_accessor_.GetStorageMode(); } + DbAccessor *SubgraphDbAccessor::GetAccessor() { return &db_accessor_; } VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; } diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index f616dc5a2..7b776f798 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -42,6 +42,8 @@ class EdgeAccessor final { explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {} + bool IsDeleted() const { return impl_.IsDeleted(); } + bool IsVisible(storage::View view) const { return impl_.IsVisible(view); } storage::EdgeTypeId EdgeType() const { return impl_.EdgeType(); } @@ -543,7 +545,7 @@ class DbAccessor final { void Abort() { accessor_->Abort(); } - storage::StorageMode GetStorageMode() const { return accessor_->GetCreationStorageMode(); } + storage::StorageMode GetStorageMode() const noexcept { return accessor_->GetCreationStorageMode(); } bool LabelIndexExists(storage::LabelId label) const { return accessor_->LabelIndexExists(label); } @@ -693,6 +695,8 @@ class SubgraphDbAccessor final { Graph *getGraph(); + storage::StorageMode GetStorageMode() const noexcept; + DbAccessor *GetAccessor(); }; diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index 940aec710..f4f3126cd 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -29,8 +29,10 @@ #include "query/frontend/ast/ast.hpp" #include "query/frontend/semantic/symbol_table.hpp" #include "query/interpret/frame.hpp" +#include "query/procedure/mg_procedure_impl.hpp" #include "query/typed_value.hpp" #include "spdlog/spdlog.h" +#include "storage/v2/storage_mode.hpp" #include "utils/exceptions.hpp" #include "utils/frame_change_id.hpp" #include "utils/logging.hpp" @@ -840,6 +842,8 @@ class ExpressionEvaluator : public ExpressionVisitor { TypedValue Visit(Function &function) override { FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_}; + bool is_transactional = storage::IsTransactional(dba_->GetStorageMode()); + TypedValue res(ctx_->memory); // Stack allocate evaluated arguments when there's a small number of them. if (function.arguments_.size() <= 8) { TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory), @@ -848,19 +852,20 @@ class ExpressionEvaluator : public ExpressionVisitor { for (size_t i = 0; i < function.arguments_.size(); ++i) { arguments[i] = function.arguments_[i]->Accept(*this); } - auto res = function.function_(arguments, function.arguments_.size(), function_ctx); - MG_ASSERT(res.GetMemoryResource() == ctx_->memory); - return res; + res = function.function_(arguments, function.arguments_.size(), function_ctx); } else { TypedValue::TVector arguments(ctx_->memory); arguments.reserve(function.arguments_.size()); for (const auto &argument : function.arguments_) { arguments.emplace_back(argument->Accept(*this)); } - auto res = function.function_(arguments.data(), arguments.size(), function_ctx); - MG_ASSERT(res.GetMemoryResource() == ctx_->memory); - return res; + res = function.function_(arguments.data(), arguments.size(), function_ctx); } + MG_ASSERT(res.GetMemoryResource() == ctx_->memory); + if (!is_transactional && res.ContainsDeleted()) [[unlikely]] { + return TypedValue(ctx_->memory); + } + return res; } TypedValue Visit(Reduce &reduce) override { diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 79a4f7d75..82269ca27 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -4835,6 +4835,12 @@ class CallProcedureCursor : public Cursor { AbortCheck(context); + auto skip_rows_with_deleted_values = [this]() { + while (result_row_it_ != result_->rows.end() && result_row_it_->has_deleted_values) { + ++result_row_it_; + } + }; + // We need to fetch new procedure results after pulling from input. // TODO: Look into openCypher's distinction between procedures returning an // empty result set vs procedures which return `void`. We currently don't @@ -4844,7 +4850,7 @@ class CallProcedureCursor : public Cursor { // It might be a good idea to resolve the procedure name once, at the // start. Unfortunately, this could deadlock if we tried to invoke a // procedure from a module (read lock) and reload a module (write lock) - // inside the same execution thread. Also, our RWLock is setup so that + // inside the same execution thread. Also, our RWLock is set up so that // it's not possible for a single thread to request multiple read locks. // Builtin module registration in query/procedure/module.cpp depends on // this locking scheme. @@ -4892,6 +4898,7 @@ class CallProcedureCursor : public Cursor { graph_view); result_->signature = &proc->results; + result_->is_transactional = storage::IsTransactional(context.db_accessor->GetStorageMode()); // Use special memory as invoking procedure is complex // TODO: This will probably need to be changed when we add support for @@ -4916,6 +4923,9 @@ class CallProcedureCursor : public Cursor { throw QueryRuntimeException("{}: {}", self_->procedure_name_, *result_->error_msg); } result_row_it_ = result_->rows.begin(); + if (!result_->is_transactional) { + skip_rows_with_deleted_values(); + } stream_exhausted = result_row_it_ == result_->rows.end(); } @@ -4945,6 +4955,9 @@ class CallProcedureCursor : public Cursor { } } ++result_row_it_; + if (!result_->is_transactional) { + skip_rows_with_deleted_values(); + } return true; } diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index ab2b3ae4b..2eea1cecb 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -32,6 +32,7 @@ #include "query/procedure/mg_procedure_helpers.hpp" #include "query/stream/common.hpp" #include "storage/v2/property_value.hpp" +#include "storage/v2/storage_mode.hpp" #include "storage/v2/view.hpp" #include "utils/algorithm.hpp" #include "utils/concepts.hpp" @@ -321,6 +322,53 @@ mgp_value_type FromTypedValueType(memgraph::query::TypedValue::Type type) { } } // namespace +bool IsDeleted(const mgp_vertex *vertex) { return vertex->getImpl().impl_.vertex_->deleted; } + +bool IsDeleted(const mgp_edge *edge) { return edge->impl.IsDeleted(); } + +bool ContainsDeleted(const mgp_path *path) { + return std::ranges::any_of(path->vertices, [](const auto &vertex) { return IsDeleted(&vertex); }) || + std::ranges::any_of(path->edges, [](const auto &edge) { return IsDeleted(&edge); }); +} + +bool ContainsDeleted(const mgp_list *list) { + return std::ranges::any_of(list->elems, [](const auto &elem) { return ContainsDeleted(&elem); }); +} + +bool ContainsDeleted(const mgp_map *map) { + return std::ranges::any_of(map->items, [](const auto &item) { return ContainsDeleted(&item.second); }); +} + +bool ContainsDeleted(const mgp_value *val) { + switch (val->type) { + // Value types + case MGP_VALUE_TYPE_NULL: + case MGP_VALUE_TYPE_BOOL: + case MGP_VALUE_TYPE_INT: + case MGP_VALUE_TYPE_DOUBLE: + case MGP_VALUE_TYPE_STRING: + case MGP_VALUE_TYPE_DATE: + case MGP_VALUE_TYPE_LOCAL_TIME: + case MGP_VALUE_TYPE_LOCAL_DATE_TIME: + case MGP_VALUE_TYPE_DURATION: + return false; + // Reference types + case MGP_VALUE_TYPE_LIST: + return ContainsDeleted(val->list_v); + case MGP_VALUE_TYPE_MAP: + return ContainsDeleted(val->map_v); + case MGP_VALUE_TYPE_VERTEX: + return IsDeleted(val->vertex_v); + case MGP_VALUE_TYPE_EDGE: + return IsDeleted(val->edge_v); + case MGP_VALUE_TYPE_PATH: + return ContainsDeleted(val->path_v); + default: + throw memgraph::query::QueryRuntimeException("Value of unknown type"); + } + return false; +} + memgraph::query::TypedValue ToTypedValue(const mgp_value &val, memgraph::utils::MemoryResource *memory) { switch (val.type) { case MGP_VALUE_TYPE_NULL: @@ -1003,6 +1051,10 @@ mgp_error mgp_list_copy(mgp_list *list, mgp_memory *memory, mgp_list **result) { void mgp_list_destroy(mgp_list *list) { DeleteRawMgpObject(list); } +mgp_error mgp_list_contains_deleted(mgp_list *list, int *result) { + return WrapExceptions([list, result] { *result = ContainsDeleted(list); }); +} + namespace { void MgpListAppendExtend(mgp_list &list, const mgp_value &value) { list.elems.push_back(value); } } // namespace @@ -1054,6 +1106,10 @@ mgp_error mgp_map_copy(mgp_map *map, mgp_memory *memory, mgp_map **result) { void mgp_map_destroy(mgp_map *map) { DeleteRawMgpObject(map); } +mgp_error mgp_map_contains_deleted(mgp_map *map, int *result) { + return WrapExceptions([map, result] { *result = ContainsDeleted(map); }); +} + mgp_error mgp_map_insert(mgp_map *map, const char *key, mgp_value *value) { return WrapExceptions([&] { auto emplace_result = map->items.emplace(key, *value); @@ -1177,6 +1233,10 @@ mgp_error mgp_path_copy(mgp_path *path, mgp_memory *memory, mgp_path **result) { void mgp_path_destroy(mgp_path *path) { DeleteRawMgpObject(path); } +mgp_error mgp_path_contains_deleted(mgp_path *path, int *result) { + return WrapExceptions([path, result] { *result = ContainsDeleted(path); }); +} + mgp_error mgp_path_expand(mgp_path *path, mgp_edge *edge) { return WrapExceptions([path, edge] { MG_ASSERT(Call(mgp_path_size, path) == path->vertices.size() - 1, "Invalid mgp_path"); @@ -1560,8 +1620,9 @@ mgp_error mgp_result_new_record(mgp_result *res, mgp_result_record **result) { auto *memory = res->rows.get_allocator().GetMemoryResource(); MG_ASSERT(res->signature, "Expected to have a valid signature"); res->rows.push_back(mgp_result_record{ - res->signature, - memgraph::utils::pmr::map(memory)}); + .signature = res->signature, + .values = memgraph::utils::pmr::map(memory), + .ignore_deleted_values = !res->is_transactional}); return &res->rows.back(); }, result); @@ -1576,10 +1637,14 @@ mgp_error mgp_result_record_insert(mgp_result_record *record, const char *field_ if (find_it == record->signature->end()) { throw std::out_of_range{fmt::format("The result doesn't have any field named '{}'.", field_name)}; } + if (record->ignore_deleted_values && ContainsDeleted(val)) [[unlikely]] { + record->has_deleted_values = true; + return; + } const auto *type = find_it->second.first; if (!type->SatisfiesType(*val)) { throw std::logic_error{ - fmt::format("The type of value doesn't satisfies the type '{}'!", type->GetPresentableName())}; + fmt::format("The type of value doesn't satisfy the type '{}'!", type->GetPresentableName())}; } record->values.emplace(field_name, ToTypedValue(*val, memory)); }); @@ -1746,7 +1811,7 @@ memgraph::storage::PropertyValue ToPropertyValue(const mgp_value &value) { return memgraph::storage::PropertyValue{memgraph::storage::TemporalData{memgraph::storage::TemporalType::Duration, value.duration_v->duration.microseconds}}; case MGP_VALUE_TYPE_VERTEX: - throw ValueConversionException{"A vertex is not a valid property value! "}; + throw ValueConversionException{"A vertex is not a valid property value!"}; case MGP_VALUE_TYPE_EDGE: throw ValueConversionException{"An edge is not a valid property value!"}; case MGP_VALUE_TYPE_PATH: @@ -1962,6 +2027,10 @@ mgp_error mgp_vertex_copy(mgp_vertex *v, mgp_memory *memory, mgp_vertex **result void mgp_vertex_destroy(mgp_vertex *v) { DeleteRawMgpObject(v); } +mgp_error mgp_vertex_is_deleted(mgp_vertex *v, int *result) { + return WrapExceptions([v] { return IsDeleted(v); }, result); +} + mgp_error mgp_vertex_equal(mgp_vertex *v1, mgp_vertex *v2, int *result) { // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) static_assert(noexcept(*v1 == *v2)); @@ -2319,6 +2388,10 @@ mgp_error mgp_edge_copy(mgp_edge *e, mgp_memory *memory, mgp_edge **result) { void mgp_edge_destroy(mgp_edge *e) { DeleteRawMgpObject(e); } +mgp_error mgp_edge_is_deleted(mgp_edge *e, int *result) { + return WrapExceptions([e] { return IsDeleted(e); }, result); +} + mgp_error mgp_edge_equal(mgp_edge *e1, mgp_edge *e2, int *result) { // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) static_assert(noexcept(*e1 == *e2)); @@ -2864,6 +2937,11 @@ mgp_error mgp_list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory, }); } +mgp_error mgp_graph_is_transactional(mgp_graph *graph, int *result) { + *result = IsTransactional(graph->storage_mode) ? 1 : 0; + return mgp_error::MGP_ERROR_NO_ERROR; +} + mgp_error mgp_graph_is_mutable(mgp_graph *graph, int *result) { *result = MgpGraphIsMutable(*graph) ? 1 : 0; return mgp_error::MGP_ERROR_NO_ERROR; diff --git a/src/query/procedure/mg_procedure_impl.hpp b/src/query/procedure/mg_procedure_impl.hpp index 7b5301381..17cac4eca 100644 --- a/src/query/procedure/mg_procedure_impl.hpp +++ b/src/query/procedure/mg_procedure_impl.hpp @@ -560,23 +560,24 @@ struct mgp_graph { // TODO: Merge `mgp_graph` and `mgp_memory` into a single `mgp_context`. The // `ctx` field is out of place here. memgraph::query::ExecutionContext *ctx; + memgraph::storage::StorageMode storage_mode; static mgp_graph WritableGraph(memgraph::query::DbAccessor &acc, memgraph::storage::View view, memgraph::query::ExecutionContext &ctx) { - return mgp_graph{&acc, view, &ctx}; + return mgp_graph{&acc, view, &ctx, acc.GetStorageMode()}; } static mgp_graph NonWritableGraph(memgraph::query::DbAccessor &acc, memgraph::storage::View view) { - return mgp_graph{&acc, view, nullptr}; + return mgp_graph{&acc, view, nullptr, acc.GetStorageMode()}; } static mgp_graph WritableGraph(memgraph::query::SubgraphDbAccessor &acc, memgraph::storage::View view, memgraph::query::ExecutionContext &ctx) { - return mgp_graph{&acc, view, &ctx}; + return mgp_graph{&acc, view, &ctx, acc.GetStorageMode()}; } static mgp_graph NonWritableGraph(memgraph::query::SubgraphDbAccessor &acc, memgraph::storage::View view) { - return mgp_graph{&acc, view, nullptr}; + return mgp_graph{&acc, view, nullptr, acc.GetStorageMode()}; } }; @@ -585,6 +586,8 @@ struct mgp_result_record { const memgraph::utils::pmr::map> *signature; memgraph::utils::pmr::map values; + bool ignore_deleted_values = false; + bool has_deleted_values = false; }; struct mgp_result { @@ -599,6 +602,7 @@ struct mgp_result { std::pair> *signature; memgraph::utils::pmr::vector rows; std::optional error_msg; + bool is_transactional = true; }; struct mgp_func_result { @@ -614,6 +618,7 @@ struct mgp_func_context { memgraph::query::DbAccessor *impl; memgraph::storage::View view; }; + struct mgp_properties_iterator { using allocator_type = memgraph::utils::Allocator; @@ -724,6 +729,7 @@ struct ProcedureInfo { bool is_batched{false}; std::optional required_privilege = std::nullopt; }; + struct mgp_proc { using allocator_type = memgraph::utils::Allocator; @@ -984,4 +990,6 @@ struct mgp_messages { storage_type messages; }; +bool ContainsDeleted(const mgp_value *val); + memgraph::query::TypedValue ToTypedValue(const mgp_value &val, memgraph::utils::MemoryResource *memory); diff --git a/src/query/procedure/py_module.cpp b/src/query/procedure/py_module.cpp index 1e687a91e..3fd09b0ab 100644 --- a/src/query/procedure/py_module.cpp +++ b/src/query/procedure/py_module.cpp @@ -25,6 +25,7 @@ #include "query/exceptions.hpp" #include "query/procedure/mg_procedure_helpers.hpp" #include "query/procedure/mg_procedure_impl.hpp" +#include "storage/v2/storage_mode.hpp" #include "utils/memory.hpp" #include "utils/on_scope_exit.hpp" #include "utils/pmr/vector.hpp" @@ -867,7 +868,37 @@ py::Object MgpListToPyTuple(mgp_list *list, PyObject *py_graph) { } namespace { -std::optional AddRecordFromPython(mgp_result *result, py::Object py_record, mgp_memory *memory) { +struct RecordFieldCache { + PyObject *key; + PyObject *val; + const char *field_name; + mgp_value *field_val; +}; + +std::optional InsertField(PyObject *key, PyObject *val, mgp_result_record *record, + const char *field_name, mgp_value *field_val) { + if (mgp_result_record_insert(record, field_name, field_val) != mgp_error::MGP_ERROR_NO_ERROR) { + std::stringstream ss; + ss << "Unable to insert field '" << py::Object::FromBorrow(key) << "' with value: '" << py::Object::FromBorrow(val) + << "'; did you set the correct field type?"; + const auto &msg = ss.str(); + PyErr_SetString(PyExc_ValueError, msg.c_str()); + mgp_value_destroy(field_val); + return py::FetchError(); + } + mgp_value_destroy(field_val); + return std::nullopt; +} + +void SkipRecord(mgp_value *field_val, std::vector ¤t_record_cache) { + mgp_value_destroy(field_val); + for (auto &cache_entry : current_record_cache) { + mgp_value_destroy(cache_entry.field_val); + } +} + +std::optional AddRecordFromPython(mgp_result *result, py::Object py_record, mgp_graph *graph, + mgp_memory *memory) { py::Object py_mgp(PyImport_ImportModule("mgp")); if (!py_mgp) return py::FetchError(); auto record_cls = py_mgp.GetAttr("Record"); @@ -888,15 +919,27 @@ std::optional AddRecordFromPython(mgp_result *result, py::Obj py::Object items(PyDict_Items(fields.Ptr())); if (!items) return py::FetchError(); mgp_result_record *record{nullptr}; - if (RaiseExceptionFromErrorCode(mgp_result_new_record(result, &record))) { - return py::FetchError(); + const auto is_transactional = storage::IsTransactional(graph->storage_mode); + if (is_transactional) { + // IN_MEMORY_ANALYTICAL must first verify that the record contains no deleted values + if (RaiseExceptionFromErrorCode(mgp_result_new_record(result, &record))) { + return py::FetchError(); + } } + std::vector current_record_cache{}; + + utils::OnScopeExit clear_record_cache{[¤t_record_cache] { + for (auto &record : current_record_cache) { + mgp_value_destroy(record.field_val); + } + }}; + Py_ssize_t len = PyList_GET_SIZE(items.Ptr()); for (Py_ssize_t i = 0; i < len; ++i) { auto *item = PyList_GET_ITEM(items.Ptr(), i); if (!item) return py::FetchError(); MG_ASSERT(PyTuple_Check(item)); - auto *key = PyTuple_GetItem(item, 0); + PyObject *key = PyTuple_GetItem(item, 0); if (!key) return py::FetchError(); if (!PyUnicode_Check(key)) { std::stringstream ss; @@ -905,30 +948,48 @@ std::optional AddRecordFromPython(mgp_result *result, py::Obj PyErr_SetString(PyExc_TypeError, msg.c_str()); return py::FetchError(); } - const auto *field_name = PyUnicode_AsUTF8(key); + const char *field_name = PyUnicode_AsUTF8(key); if (!field_name) return py::FetchError(); - auto *val = PyTuple_GetItem(item, 1); + PyObject *val = PyTuple_GetItem(item, 1); if (!val) return py::FetchError(); // This memory is one dedicated for mg_procedure. mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, memory); if (field_val == nullptr) { return py::FetchError(); } - if (mgp_result_record_insert(record, field_name, field_val) != mgp_error::MGP_ERROR_NO_ERROR) { - std::stringstream ss; - ss << "Unable to insert field '" << py::Object::FromBorrow(key) << "' with value: '" - << py::Object::FromBorrow(val) << "'; did you set the correct field type?"; - const auto &msg = ss.str(); - PyErr_SetString(PyExc_ValueError, msg.c_str()); - mgp_value_destroy(field_val); - return py::FetchError(); + + if (!is_transactional) { + // If a deleted value is being inserted into a record, skip the whole record + if (ContainsDeleted(field_val)) { + SkipRecord(field_val, current_record_cache); + return std::nullopt; + } + current_record_cache.emplace_back( + RecordFieldCache{.key = key, .val = val, .field_name = field_name, .field_val = field_val}); + } else { + auto maybe_exc = InsertField(key, val, record, field_name, field_val); + if (maybe_exc) return maybe_exc; } - mgp_value_destroy(field_val); } + + if (is_transactional) { + return std::nullopt; + } + + // IN_MEMORY_ANALYTICAL only adds a new record after verifying that it contains no deleted values + if (RaiseExceptionFromErrorCode(mgp_result_new_record(result, &record))) { + return py::FetchError(); + } + for (auto &cache_entry : current_record_cache) { + auto maybe_exc = + InsertField(cache_entry.key, cache_entry.val, record, cache_entry.field_name, cache_entry.field_val); + if (maybe_exc) return maybe_exc; + } + return std::nullopt; } -std::optional AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq, +std::optional AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq, mgp_graph *graph, mgp_memory *memory) { Py_ssize_t len = PySequence_Size(py_seq.Ptr()); if (len == -1) return py::FetchError(); @@ -938,7 +999,7 @@ std::optional AddMultipleRecordsFromPython(mgp_result *result for (Py_ssize_t i = 0, curr_item = 0; i < len; ++i, ++curr_item) { py::Object py_record(PySequence_GetItem(py_seq.Ptr(), curr_item)); if (!py_record) return py::FetchError(); - auto maybe_exc = AddRecordFromPython(result, py_record, memory); + auto maybe_exc = AddRecordFromPython(result, py_record, graph, memory); if (maybe_exc) return maybe_exc; // Once PySequence_DelSlice deletes "transformed" objects, starting index is 0 again. if (i && i % del_cnt == 0) { @@ -952,14 +1013,14 @@ std::optional AddMultipleRecordsFromPython(mgp_result *result } std::optional AddMultipleBatchRecordsFromPython(mgp_result *result, py::Object py_seq, - mgp_memory *memory) { + mgp_graph *graph, mgp_memory *memory) { Py_ssize_t len = PySequence_Size(py_seq.Ptr()); if (len == -1) return py::FetchError(); result->rows.reserve(len); for (Py_ssize_t i = 0; i < len; ++i) { py::Object py_record(PySequence_GetItem(py_seq.Ptr(), i)); if (!py_record) return py::FetchError(); - auto maybe_exc = AddRecordFromPython(result, py_record, memory); + auto maybe_exc = AddRecordFromPython(result, py_record, graph, memory); if (maybe_exc) return maybe_exc; } PySequence_DelSlice(py_seq.Ptr(), 0, PySequence_Size(py_seq.Ptr())); @@ -1015,11 +1076,11 @@ void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *gra if (!py_res) return py::FetchError(); if (PySequence_Check(py_res.Ptr())) { if (is_batched) { - return AddMultipleBatchRecordsFromPython(result, py_res, memory); + return AddMultipleBatchRecordsFromPython(result, py_res, graph, memory); } - return AddMultipleRecordsFromPython(result, py_res, memory); + return AddMultipleRecordsFromPython(result, py_res, graph, memory); } - return AddRecordFromPython(result, py_res, memory); + return AddRecordFromPython(result, py_res, graph, memory); }; // It is *VERY IMPORTANT* to note that this code takes great care not to keep @@ -1114,9 +1175,9 @@ void CallPythonTransformation(const py::Object &py_cb, mgp_messages *msgs, mgp_g auto py_res = py_cb.Call(py_graph, py_messages); if (!py_res) return py::FetchError(); if (PySequence_Check(py_res.Ptr())) { - return AddMultipleRecordsFromPython(result, py_res, memory); + return AddMultipleRecordsFromPython(result, py_res, graph, memory); } - return AddRecordFromPython(result, py_res, memory); + return AddRecordFromPython(result, py_res, graph, memory); }; // It is *VERY IMPORTANT* to note that this code takes great care not to keep @@ -1164,9 +1225,27 @@ void CallPythonFunction(const py::Object &py_cb, mgp_list *args, mgp_graph *grap auto call = [&](py::Object py_graph) -> utils::BasicResult, mgp_value *> { py::Object py_args(MgpListToPyTuple(args, py_graph.Ptr())); if (!py_args) return {py::FetchError()}; + const auto is_transactional = storage::IsTransactional(graph->storage_mode); auto py_res = py_cb.Call(py_graph, py_args); if (!py_res) return {py::FetchError()}; mgp_value *ret_val = PyObjectToMgpValueWithPythonExceptions(py_res.Ptr(), memory); + if (!is_transactional && ContainsDeleted(ret_val)) { + mgp_value_destroy(ret_val); + mgp_value *null_val{nullptr}; + mgp_error last_error{mgp_error::MGP_ERROR_NO_ERROR}; + + last_error = mgp_value_make_null(memory, &null_val); + + if (last_error == mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { + throw std::bad_alloc{}; + } + if (last_error != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::runtime_error{"Unexpected error while creating mgp_value"}; + } + + return null_val; + } + if (ret_val == nullptr) { return {py::FetchError()}; } diff --git a/src/query/stream/streams.cpp b/src/query/stream/streams.cpp index e93bf06a7..896607e94 100644 --- a/src/query/stream/streams.cpp +++ b/src/query/stream/streams.cpp @@ -99,7 +99,7 @@ void CallCustomTransformation(const std::string &transformation_name, const std: mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}}; std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages), [](const TMessage &message) { return mgp_message{message}; }); - mgp_graph graph{&db_accessor, storage::View::OLD, nullptr}; + mgp_graph graph{&db_accessor, storage::View::OLD, nullptr, db_accessor.GetStorageMode()}; mgp_memory memory{&memory_resource}; result.rows.clear(); result.error_msg.reset(); diff --git a/src/query/typed_value.cpp b/src/query/typed_value.cpp index 91893d71c..ea883e428 100644 --- a/src/query/typed_value.cpp +++ b/src/query/typed_value.cpp @@ -370,6 +370,38 @@ bool TypedValue::IsGraph() const { return type_ == Type::Graph; } #undef DEFINE_VALUE_AND_TYPE_GETTERS +bool TypedValue::ContainsDeleted() const { + switch (type_) { + // Value types + case Type::Null: + case Type::Bool: + case Type::Int: + case Type::Double: + case Type::String: + case Type::Date: + case Type::LocalTime: + case Type::LocalDateTime: + case Type::Duration: + return false; + // Reference types + case Type::List: + return std::ranges::any_of(list_v, [](const auto &elem) { return elem.ContainsDeleted(); }); + case Type::Map: + return std::ranges::any_of(map_v, [](const auto &item) { return item.second.ContainsDeleted(); }); + case Type::Vertex: + return vertex_v.impl_.vertex_->deleted; + case Type::Edge: + return edge_v.IsDeleted(); + case Type::Path: + return std::ranges::any_of(path_v.vertices(), + [](auto &vertex_acc) { return vertex_acc.impl_.vertex_->deleted; }) || + std::ranges::any_of(path_v.edges(), [](auto &edge_acc) { return edge_acc.IsDeleted(); }); + default: + throw TypedValueException("Value of unknown type"); + } + return false; +} + bool TypedValue::IsNull() const { return type_ == Type::Null; } bool TypedValue::IsNumeric() const { return IsInt() || IsDouble(); } diff --git a/src/query/typed_value.hpp b/src/query/typed_value.hpp index 0af38cecc..a1353869a 100644 --- a/src/query/typed_value.hpp +++ b/src/query/typed_value.hpp @@ -515,6 +515,8 @@ class TypedValue { #undef DECLARE_VALUE_AND_TYPE_GETTERS + bool ContainsDeleted() const; + /** Checks if value is a TypedValue::Null. */ bool IsNull() const; diff --git a/src/storage/v2/edge_accessor.cpp b/src/storage/v2/edge_accessor.cpp index 3d97f537a..7e7166117 100644 --- a/src/storage/v2/edge_accessor.cpp +++ b/src/storage/v2/edge_accessor.cpp @@ -25,6 +25,13 @@ namespace memgraph::storage { +bool EdgeAccessor::IsDeleted() const { + if (!storage_->config_.items.properties_on_edges) { + return false; + } + return edge_.ptr->deleted; +} + bool EdgeAccessor::IsVisible(const View view) const { bool exists = true; bool deleted = true; diff --git a/src/storage/v2/edge_accessor.hpp b/src/storage/v2/edge_accessor.hpp index 365619149..a1c52d0a5 100644 --- a/src/storage/v2/edge_accessor.hpp +++ b/src/storage/v2/edge_accessor.hpp @@ -44,6 +44,8 @@ class EdgeAccessor final { transaction_(transaction), for_deleted_(for_deleted) {} + bool IsDeleted() const; + /// @return true if the object is visible from the current transaction bool IsVisible(View view) const; diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 6104ea6c7..74f9acb92 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -85,7 +85,7 @@ Storage::Accessor::Accessor(Accessor &&other) noexcept other.commit_timestamp_.reset(); } -StorageMode Storage::GetStorageMode() const { return storage_mode_; } +StorageMode Storage::GetStorageMode() const noexcept { return storage_mode_; } IsolationLevel Storage::GetIsolationLevel() const noexcept { return isolation_level_; } @@ -95,7 +95,7 @@ utils::BasicResult Storage::SetIsolationLevel(I return {}; } -StorageMode Storage::Accessor::GetCreationStorageMode() const { return creation_storage_mode_; } +StorageMode Storage::Accessor::GetCreationStorageMode() const noexcept { return creation_storage_mode_; } std::optional Storage::Accessor::GetTransactionId() const { if (is_transaction_active_) { diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 60752d101..14caf4b43 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -238,7 +238,7 @@ class Storage { EdgeTypeId NameToEdgeType(std::string_view name) { return storage_->NameToEdgeType(name); } - StorageMode GetCreationStorageMode() const; + StorageMode GetCreationStorageMode() const noexcept; const std::string &id() const { return storage_->id(); } @@ -304,7 +304,7 @@ class Storage { return EdgeTypeId::FromUint(name_id_mapper_->NameToId(name)); } - StorageMode GetStorageMode() const; + StorageMode GetStorageMode() const noexcept; virtual void FreeMemory(std::unique_lock main_guard) = 0; diff --git a/src/storage/v2/storage_mode.cpp b/src/storage/v2/storage_mode.cpp index 73a886d6a..067646854 100644 --- a/src/storage/v2/storage_mode.cpp +++ b/src/storage/v2/storage_mode.cpp @@ -13,6 +13,10 @@ namespace memgraph::storage { +bool IsTransactional(const StorageMode storage_mode) noexcept { + return storage_mode != StorageMode::IN_MEMORY_ANALYTICAL; +} + std::string_view StorageModeToString(memgraph::storage::StorageMode storage_mode) { switch (storage_mode) { case memgraph::storage::StorageMode::IN_MEMORY_ANALYTICAL: diff --git a/src/storage/v2/storage_mode.hpp b/src/storage/v2/storage_mode.hpp index 2ab348c59..c02d3c177 100644 --- a/src/storage/v2/storage_mode.hpp +++ b/src/storage/v2/storage_mode.hpp @@ -18,6 +18,8 @@ namespace memgraph::storage { enum class StorageMode : std::uint8_t { IN_MEMORY_ANALYTICAL, IN_MEMORY_TRANSACTIONAL, ON_DISK_TRANSACTIONAL }; +bool IsTransactional(const StorageMode storage_mode) noexcept; + std::string_view StorageModeToString(memgraph::storage::StorageMode storage_mode); } // namespace memgraph::storage diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 594c9acb0..da3e9101a 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -72,6 +72,7 @@ add_subdirectory(constraints) add_subdirectory(inspect_query) add_subdirectory(filter_info) add_subdirectory(queries) +add_subdirectory(query_modules_storage_modes) add_subdirectory(garbage_collection) add_subdirectory(query_planning) diff --git a/tests/e2e/query_modules_storage_modes/CMakeLists.txt b/tests/e2e/query_modules_storage_modes/CMakeLists.txt new file mode 100644 index 000000000..5bd0ac436 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/CMakeLists.txt @@ -0,0 +1,8 @@ +function(copy_qm_storage_modes_e2e_python_files FILE_NAME) + copy_e2e_python_files(query_modules_storage_modes ${FILE_NAME}) +endfunction() + +copy_qm_storage_modes_e2e_python_files(common.py) +copy_qm_storage_modes_e2e_python_files(test_query_modules_storage_modes.py) + +add_subdirectory(query_modules) diff --git a/tests/e2e/query_modules_storage_modes/common.py b/tests/e2e/query_modules_storage_modes/common.py new file mode 100644 index 000000000..19abde158 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/common.py @@ -0,0 +1,37 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import typing + +import mgclient +import pytest + + +@pytest.fixture(scope="function") +def cursor(**kwargs) -> mgclient.Connection: + connection = mgclient.connect(host="localhost", port=7687, **kwargs) + connection.autocommit = True + cursor = connection.cursor() + + cursor.execute("MATCH (n) DETACH DELETE n;") + cursor.execute("CREATE (m:Component {id: 'A7422'}), (n:Component {id: '7X8X0'});") + cursor.execute("MATCH (m:Component {id: 'A7422'}) MATCH (n:Component {id: '7X8X0'}) CREATE (m)-[:PART_OF]->(n);") + cursor.execute("MATCH (m:Component {id: 'A7422'}) MATCH (n:Component {id: '7X8X0'}) CREATE (n)-[:DEPENDS_ON]->(m);") + + yield cursor + + cursor.execute("MATCH (n) DETACH DELETE n;") + + +def connect(**kwargs): + connection = mgclient.connect(host="localhost", port=7687, **kwargs) + connection.autocommit = True + return connection.cursor() diff --git a/tests/e2e/query_modules_storage_modes/query_modules/CMakeLists.txt b/tests/e2e/query_modules_storage_modes/query_modules/CMakeLists.txt new file mode 100644 index 000000000..44df43702 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/query_modules/CMakeLists.txt @@ -0,0 +1,4 @@ +copy_qm_storage_modes_e2e_python_files(python_api.py) + +add_query_module(c_api c_api.cpp) +add_query_module(cpp_api cpp_api.cpp) diff --git a/tests/e2e/query_modules_storage_modes/query_modules/c_api.cpp b/tests/e2e/query_modules_storage_modes/query_modules/c_api.cpp new file mode 100644 index 000000000..85663a829 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/query_modules/c_api.cpp @@ -0,0 +1,70 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include + +#include "_mgp.hpp" +#include "mg_exceptions.hpp" +#include "mg_procedure.h" + +constexpr std::string_view kFunctionPassRelationship = "pass_relationship"; +constexpr std::string_view kPassRelationshipArg = "relationship"; + +constexpr std::string_view kProcedurePassNodeWithId = "pass_node_with_id"; +constexpr std::string_view kPassNodeWithIdArg = "node"; +constexpr std::string_view kPassNodeWithIdFieldNode = "node"; +constexpr std::string_view kPassNodeWithIdFieldId = "id"; + +// While the query procedure/function sleeps for this amount of time, a parallel transaction will erase a graph element +// (node or relationship) contained in the return value. Any operation in the parallel transaction should take far less +// time than this value. +const int64_t kSleep = 1; + +void PassRelationship(mgp_list *args, mgp_func_context *ctx, mgp_func_result *res, mgp_memory *memory) { + auto *relationship = mgp::list_at(args, 0); + + std::this_thread::sleep_for(std::chrono::seconds(kSleep)); + + mgp::func_result_set_value(res, relationship, memory); +} + +void PassNodeWithId(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + auto *node = mgp::value_get_vertex(mgp::list_at(args, 0)); + auto node_id = mgp::vertex_get_id(node).as_int; + + std::this_thread::sleep_for(std::chrono::seconds(kSleep)); + + auto *result_record = mgp::result_new_record(result); + mgp::result_record_insert(result_record, kPassNodeWithIdFieldNode.data(), mgp::value_make_vertex(node)); + mgp::result_record_insert(result_record, kPassNodeWithIdFieldId.data(), mgp::value_make_int(node_id, memory)); +} + +extern "C" int mgp_init_module(struct mgp_module *query_module, struct mgp_memory *memory) { + try { + { + auto *func = mgp::module_add_function(query_module, kFunctionPassRelationship.data(), PassRelationship); + mgp::func_add_arg(func, kPassRelationshipArg.data(), mgp::type_relationship()); + } + { + auto *proc = mgp::module_add_read_procedure(query_module, kProcedurePassNodeWithId.data(), PassNodeWithId); + mgp::proc_add_arg(proc, kPassNodeWithIdArg.data(), mgp::type_node()); + mgp::proc_add_result(proc, kPassNodeWithIdFieldNode.data(), mgp::type_node()); + mgp::proc_add_result(proc, kPassNodeWithIdFieldId.data(), mgp::type_int()); + } + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/query_modules_storage_modes/query_modules/cpp_api.cpp b/tests/e2e/query_modules_storage_modes/query_modules/cpp_api.cpp new file mode 100644 index 000000000..a74dce832 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/query_modules/cpp_api.cpp @@ -0,0 +1,86 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include + +#include + +constexpr std::string_view kFunctionPassRelationship = "pass_relationship"; +constexpr std::string_view kPassRelationshipArg = "relationship"; + +constexpr std::string_view kProcedurePassNodeWithId = "pass_node_with_id"; +constexpr std::string_view kPassNodeWithIdArg = "node"; +constexpr std::string_view kPassNodeWithIdFieldNode = "node"; +constexpr std::string_view kPassNodeWithIdFieldId = "id"; + +// While the query procedure/function sleeps for this amount of time, a parallel transaction will erase a graph element +// (node or relationship) contained in the return value. Any operation in the parallel transaction should take far less +// time than this value. +const int64_t kSleep = 1; + +void PassRelationship(mgp_list *args, mgp_func_context *ctx, mgp_func_result *res, mgp_memory *memory) { + try { + mgp::MemoryDispatcherGuard guard{memory}; + const auto arguments = mgp::List(args); + auto result = mgp::Result(res); + + const auto relationship = arguments[0].ValueRelationship(); + + std::this_thread::sleep_for(std::chrono::seconds(kSleep)); + + result.SetValue(relationship); + } catch (const std::exception &e) { + mgp::func_result_set_error_msg(res, e.what(), memory); + return; + } +} + +void PassNodeWithId(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + try { + mgp::MemoryDispatcherGuard guard(memory); + const auto arguments = mgp::List(args); + const auto record_factory = mgp::RecordFactory(result); + + const auto node = arguments[0].ValueNode(); + const auto node_id = node.Id().AsInt(); + + std::this_thread::sleep_for(std::chrono::seconds(kSleep)); + + auto record = record_factory.NewRecord(); + record.Insert(kPassNodeWithIdFieldNode.data(), node); + record.Insert(kPassNodeWithIdFieldId.data(), node_id); + } catch (const std::exception &e) { + mgp::result_set_error_msg(result, e.what()); + return; + } +} + +extern "C" int mgp_init_module(struct mgp_module *query_module, struct mgp_memory *memory) { + try { + mgp::MemoryDispatcherGuard guard(memory); + + mgp::AddFunction(PassRelationship, kFunctionPassRelationship, + {mgp::Parameter(kPassRelationshipArg, mgp::Type::Relationship)}, query_module, memory); + + mgp::AddProcedure( + PassNodeWithId, kProcedurePassNodeWithId, mgp::ProcedureType::Read, + {mgp::Parameter(kPassNodeWithIdArg, mgp::Type::Node)}, + {mgp::Return(kPassNodeWithIdFieldNode, mgp::Type::Node), mgp::Return(kPassNodeWithIdFieldId, mgp::Type::Int)}, + query_module, memory); + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/query_modules_storage_modes/query_modules/python_api.py b/tests/e2e/query_modules_storage_modes/query_modules/python_api.py new file mode 100644 index 000000000..90db4c97d --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/query_modules/python_api.py @@ -0,0 +1,55 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +from time import sleep + +import mgp + +# While the query procedure/function sleeps for this amount of time, a parallel transaction will erase a graph element +# (node or relationship) contained in the return value. Any operation in the parallel transaction should take far less +# time than this value. +SLEEP = 1 + + +@mgp.read_proc +def pass_node_with_id(ctx: mgp.ProcCtx, node: mgp.Vertex) -> mgp.Record(node=mgp.Vertex, id=int): + sleep(SLEEP) + return mgp.Record(node=node, id=node.id) + + +@mgp.function +def pass_node(ctx: mgp.FuncCtx, node: mgp.Vertex): + sleep(SLEEP) + return node + + +@mgp.function +def pass_relationship(ctx: mgp.FuncCtx, relationship: mgp.Edge): + sleep(SLEEP) + return relationship + + +@mgp.function +def pass_path(ctx: mgp.FuncCtx, path: mgp.Path): + sleep(SLEEP) + return path + + +@mgp.function +def pass_list(ctx: mgp.FuncCtx, list_: mgp.List[mgp.Any]): + sleep(SLEEP) + return list_ + + +@mgp.function +def pass_map(ctx: mgp.FuncCtx, map_: mgp.Map): + sleep(SLEEP) + return map_ diff --git a/tests/e2e/query_modules_storage_modes/test_query_modules_storage_modes.py b/tests/e2e/query_modules_storage_modes/test_query_modules_storage_modes.py new file mode 100644 index 000000000..2c6eeac20 --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/test_query_modules_storage_modes.py @@ -0,0 +1,283 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# isort: off +from multiprocessing import Process +import sys +import pytest + +from common import cursor, connect + +import time + +SWITCH_TO_ANALYTICAL = "STORAGE MODE IN_MEMORY_ANALYTICAL;" + + +def modify_graph(query): + subprocess_cursor = connect() + + time.sleep(0.5) # Time for the parallel transaction to call a query procedure + + subprocess_cursor.execute(query) + + +@pytest.mark.parametrize("api", ["c", "cpp", "python"]) +def test_function_delete_result(cursor, api): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (m:Component {id: 'A7422'})-[e:PART_OF]->(n:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute(f"MATCH (m)-[e]->(n) RETURN {api}_api.pass_relationship(e);") + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0].type == "DEPENDS_ON" + + +@pytest.mark.parametrize("api", ["c", "cpp", "python"]) +def test_function_delete_only_result(cursor, api): + cursor.execute(SWITCH_TO_ANALYTICAL) + + cursor.execute("MATCH (m:Component {id: '7X8X0'})-[e:DEPENDS_ON]->(n:Component {id: 'A7422'}) DELETE e;") + + deleter = Process( + target=modify_graph, + args=("MATCH (m:Component {id: 'A7422'})-[e:PART_OF]->(n:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute(f"MATCH (m)-[e]->(n) RETURN {api}_api.pass_relationship(e);") + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +@pytest.mark.parametrize("api", ["c", "cpp", "python"]) +def test_procedure_delete_result(cursor, api): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (n {id: 'A7422'}) DETACH DELETE n;",), + ) + deleter.start() + + cursor.execute( + f"""MATCH (n) + CALL {api}_api.pass_node_with_id(n) + YIELD node, id + RETURN node, id;""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 2 and result[0][0].properties["id"] == "7X8X0" + + +@pytest.mark.parametrize("api", ["c", "cpp", "python"]) +def test_procedure_delete_only_result(cursor, api): + cursor.execute(SWITCH_TO_ANALYTICAL) + + cursor.execute("MATCH (n {id: '7X8X0'}) DETACH DELETE n;") + + deleter = Process( + target=modify_graph, + args=("MATCH (n {id: 'A7422'}) DETACH DELETE n;",), + ) + deleter.start() + + cursor.execute( + f"""MATCH (n) + CALL {api}_api.pass_node_with_id(n) + YIELD node, id + RETURN node, id;""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 0 + + +def test_deleted_node(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process(target=modify_graph, args=("MATCH (n:Component {id: 'A7422'}) DETACH DELETE n;",)) + deleter.start() + + cursor.execute( + """MATCH (n: Component {id: 'A7422'}) + RETURN python_api.pass_node(n);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +def test_deleted_relationship(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (:Component {id: 'A7422'})-[e:PART_OF]->(:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute( + """MATCH (:Component {id: 'A7422'})-[e:PART_OF]->(:Component {id: '7X8X0'}) + RETURN python_api.pass_relationship(e);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +def test_deleted_node_in_path(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process(target=modify_graph, args=("MATCH (n:Component {id: 'A7422'}) DETACH DELETE n;",)) + deleter.start() + + cursor.execute( + """MATCH path=(n {id: 'A7422'})-[e]->(m) + RETURN python_api.pass_path(path);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +def test_deleted_relationship_in_path(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (:Component {id: 'A7422'})-[e:PART_OF]->(:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute( + """MATCH path=(n {id: 'A7422'})-[e]->(m) + RETURN python_api.pass_path(path);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +def test_deleted_value_in_list(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (:Component {id: 'A7422'})-[e:PART_OF]->(:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute( + """MATCH (n)-[e]->() + WITH collect(n) + collect(e) as list + RETURN python_api.pass_list(list);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +def test_deleted_value_in_map(cursor): + cursor.execute(SWITCH_TO_ANALYTICAL) + + deleter = Process( + target=modify_graph, + args=("MATCH (:Component {id: 'A7422'})-[e:PART_OF]->(:Component {id: '7X8X0'}) DELETE e;",), + ) + deleter.start() + + cursor.execute( + """MATCH (n {id: 'A7422'})-[e]->() + WITH {node: n, relationship: e} AS map + RETURN python_api.pass_map(map);""" + ) + + deleter.join() + + result = cursor.fetchall() + + assert len(result) == 1 and len(result[0]) == 1 and result[0][0] is None + + +@pytest.mark.parametrize("storage_mode", ["IN_MEMORY_TRANSACTIONAL", "IN_MEMORY_ANALYTICAL"]) +def test_function_none_deleted(storage_mode): + cursor = connect() + + cursor.execute(f"STORAGE MODE {storage_mode};") + cursor.execute("CREATE (m:Component {id: 'A7422'}), (n:Component {id: '7X8X0'});") + + cursor.execute( + """MATCH (n) + RETURN python_api.pass_node(n);""" + ) + + result = cursor.fetchall() + cursor.execute("MATCH (n) DETACH DELETE n;") + + assert len(result) == 2 + + +@pytest.mark.parametrize("storage_mode", ["IN_MEMORY_TRANSACTIONAL", "IN_MEMORY_ANALYTICAL"]) +def test_procedure_none_deleted(storage_mode): + cursor = connect() + + cursor.execute(f"STORAGE MODE {storage_mode};") + cursor.execute("CREATE (m:Component {id: 'A7422'}), (n:Component {id: '7X8X0'});") + + cursor.execute( + """MATCH (n) + CALL python_api.pass_node_with_id(n) + YIELD node, id + RETURN node, id;""" + ) + + result = cursor.fetchall() + cursor.execute("MATCH (n) DETACH DELETE n;") + + assert len(result) == 2 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/query_modules_storage_modes/workloads.yaml b/tests/e2e/query_modules_storage_modes/workloads.yaml new file mode 100644 index 000000000..d8caee2ca --- /dev/null +++ b/tests/e2e/query_modules_storage_modes/workloads.yaml @@ -0,0 +1,14 @@ +query_modules_storage_modes_cluster: &query_modules_storage_modes_cluster + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "query_modules_storage_modes.log" + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Test query module API behavior in Memgraph storage modes" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/query_modules_storage_modes/query_modules/" + args: ["query_modules_storage_modes/test_query_modules_storage_modes.py"] + <<: *query_modules_storage_modes_cluster diff --git a/tests/gql_behave/continuous_integration b/tests/gql_behave/continuous_integration index 850fc5934..26ee6d727 100755 --- a/tests/gql_behave/continuous_integration +++ b/tests/gql_behave/continuous_integration @@ -104,7 +104,7 @@ class MemgraphRunner: memgraph_binary = os.path.join(self.build_directory, "memgraph") args_mg = [ memgraph_binary, - "--storage-properties-on-edges", + "--storage-properties-on-edges=true", "--data-directory", self.data_directory.name, "--log-file", diff --git a/tests/unit/cpp_api.cpp b/tests/unit/cpp_api.cpp index 929a70431..012b2d713 100644 --- a/tests/unit/cpp_api.cpp +++ b/tests/unit/cpp_api.cpp @@ -38,7 +38,8 @@ struct CppApiTestFixture : public ::testing::Test { mgp_graph CreateGraph(const memgraph::storage::View view = memgraph::storage::View::NEW) { // the execution context can be null as it shouldn't be used in these tests - return mgp_graph{&CreateDbAccessor(memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION), view, ctx_.get()}; + return mgp_graph{&CreateDbAccessor(memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION), view, ctx_.get(), + memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL}; } memgraph::query::DbAccessor &CreateDbAccessor(const memgraph::storage::IsolationLevel isolationLevel) { @@ -499,6 +500,7 @@ TYPED_TEST(CppApiTestFixture, TestValueOperatorLessThan) { ASSERT_THROW(list_test < map_test, mgp::ValueException); ASSERT_THROW(list_test < list_test, mgp::ValueException); } + TYPED_TEST(CppApiTestFixture, TestNumberEquality) { mgp::Value double_1{1.0}; mgp::Value int_1{static_cast(1)}; diff --git a/tests/unit/query_procedure_mgp_type.cpp b/tests/unit/query_procedure_mgp_type.cpp index 4ed9f4926..c5f30f3af 100644 --- a/tests/unit/query_procedure_mgp_type.cpp +++ b/tests/unit/query_procedure_mgp_type.cpp @@ -249,7 +249,7 @@ TYPED_TEST(CypherType, VertexSatisfiesType) { auto vertex = dba.InsertVertex(); mgp_memory memory{memgraph::utils::NewDeleteResource()}; memgraph::utils::Allocator alloc(memory.impl); - mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr, dba.GetStorageMode()}; auto *mgp_vertex_v = EXPECT_MGP_NO_ERROR(mgp_value *, mgp_value_make_vertex, alloc.new_object(vertex, &graph)); const memgraph::query::TypedValue tv_vertex(vertex); @@ -274,7 +274,7 @@ TYPED_TEST(CypherType, EdgeSatisfiesType) { auto edge = *dba.InsertEdge(&v1, &v2, dba.NameToEdgeType("edge_type")); mgp_memory memory{memgraph::utils::NewDeleteResource()}; memgraph::utils::Allocator alloc(memory.impl); - mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr, dba.GetStorageMode()}; auto *mgp_edge_v = EXPECT_MGP_NO_ERROR(mgp_value *, mgp_value_make_edge, alloc.new_object(edge, &graph)); const memgraph::query::TypedValue tv_edge(edge); CheckSatisfiesTypesAndNullable( @@ -298,7 +298,7 @@ TYPED_TEST(CypherType, PathSatisfiesType) { auto edge = *dba.InsertEdge(&v1, &v2, dba.NameToEdgeType("edge_type")); mgp_memory memory{memgraph::utils::NewDeleteResource()}; memgraph::utils::Allocator alloc(memory.impl); - mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::NEW, nullptr, dba.GetStorageMode()}; auto *mgp_vertex_v = alloc.new_object(v1, &graph); auto path = EXPECT_MGP_NO_ERROR(mgp_path *, mgp_path_make_with_start, mgp_vertex_v, &memory); ASSERT_TRUE(path); diff --git a/tests/unit/query_procedure_py_module.cpp b/tests/unit/query_procedure_py_module.cpp index 9487ebb2c..dd744229a 100644 --- a/tests/unit/query_procedure_py_module.cpp +++ b/tests/unit/query_procedure_py_module.cpp @@ -132,7 +132,7 @@ TYPED_TEST(PyModule, PyVertex) { auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); mgp_memory memory{memgraph::utils::NewDeleteResource()}; - mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr, dba.GetStorageMode()}; auto *vertex = EXPECT_MGP_NO_ERROR(mgp_vertex *, mgp_graph_get_vertex_by_id, &graph, mgp_vertex_id{0}, &memory); ASSERT_TRUE(vertex); auto *vertex_value = EXPECT_MGP_NO_ERROR(mgp_value *, mgp_value_make_vertex, @@ -182,7 +182,7 @@ TYPED_TEST(PyModule, PyEdge) { auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); mgp_memory memory{memgraph::utils::NewDeleteResource()}; - mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr, dba.GetStorageMode()}; auto *start_v = EXPECT_MGP_NO_ERROR(mgp_vertex *, mgp_graph_get_vertex_by_id, &graph, mgp_vertex_id{0}, &memory); ASSERT_TRUE(start_v); auto *edges_it = EXPECT_MGP_NO_ERROR(mgp_edges_iterator *, mgp_vertex_iter_out_edges, start_v, &memory); @@ -228,7 +228,7 @@ TYPED_TEST(PyModule, PyPath) { auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); mgp_memory memory{memgraph::utils::NewDeleteResource()}; - mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr}; + mgp_graph graph{&dba, memgraph::storage::View::OLD, nullptr, dba.GetStorageMode()}; auto *start_v = EXPECT_MGP_NO_ERROR(mgp_vertex *, mgp_graph_get_vertex_by_id, &graph, mgp_vertex_id{0}, &memory); ASSERT_TRUE(start_v); auto *path = EXPECT_MGP_NO_ERROR(mgp_path *, mgp_path_make_with_start, start_v, &memory); diff --git a/tests/unit/query_procedures_mgp_graph.cpp b/tests/unit/query_procedures_mgp_graph.cpp index 207080967..785aab2cf 100644 --- a/tests/unit/query_procedures_mgp_graph.cpp +++ b/tests/unit/query_procedures_mgp_graph.cpp @@ -120,7 +120,8 @@ class MgpGraphTest : public ::testing::Test { public: mgp_graph CreateGraph(const memgraph::storage::View view = memgraph::storage::View::NEW) { // the execution context can be null as it shouldn't be used in these tests - return mgp_graph{&CreateDbAccessor(memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION), view, ctx_.get()}; + return mgp_graph{&CreateDbAccessor(memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION), view, ctx_.get(), + memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL}; } std::array CreateEdge() { From 64e5428d9410f82fbf88c2814b51f374ce0b7af1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Pu=C5=A1i=C4=87?= Date: Mon, 4 Dec 2023 10:52:00 +0100 Subject: [PATCH 38/52] Send Bolt success messages only after DB operations run successfully (#1556) --- src/communication/bolt/v1/states/handlers.hpp | 32 ++++++------ tests/drivers/python/v5_8/transactions.py | 51 ++++++++++++++++++- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/communication/bolt/v1/states/handlers.hpp b/src/communication/bolt/v1/states/handlers.hpp index f873f0e6e..3b5a67b17 100644 --- a/src/communication/bolt/v1/states/handlers.hpp +++ b/src/communication/bolt/v1/states/handlers.hpp @@ -367,14 +367,16 @@ State HandleReset(TSession &session, const Marker marker) { return State::Close; } - if (!session.encoder_.MessageSuccess()) { - spdlog::trace("Couldn't send success message!"); - return State::Close; + try { + session.Abort(); + if (!session.encoder_.MessageSuccess({})) { + spdlog::trace("Couldn't send success message!"); + return State::Close; + } + return State::Idle; + } catch (const std::exception &e) { + return HandleFailure(session, e); } - - session.Abort(); - - return State::Idle; } template @@ -397,19 +399,17 @@ State HandleBegin(TSession &session, const State state, const Marker marker) { DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state"); - if (!session.encoder_.MessageSuccess({})) { - spdlog::trace("Couldn't send success message!"); - return State::Close; - } - try { session.Configure(extra.ValueMap()); session.BeginTransaction(extra.ValueMap()); + if (!session.encoder_.MessageSuccess({})) { + spdlog::trace("Couldn't send success message!"); + return State::Close; + } + return State::Idle; } catch (const std::exception &e) { return HandleFailure(session, e); } - - return State::Idle; } template @@ -427,11 +427,11 @@ State HandleCommit(TSession &session, const State state, const Marker marker) { DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state"); try { + session.CommitTransaction(); if (!session.encoder_.MessageSuccess({})) { spdlog::trace("Couldn't send success message!"); return State::Close; } - session.CommitTransaction(); return State::Idle; } catch (const std::exception &e) { return HandleFailure(session, e); @@ -453,11 +453,11 @@ State HandleRollback(TSession &session, const State state, const Marker marker) DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state"); try { + session.RollbackTransaction(); if (!session.encoder_.MessageSuccess({})) { spdlog::trace("Couldn't send success message!"); return State::Close; } - session.RollbackTransaction(); return State::Idle; } catch (const std::exception &e) { return HandleFailure(session, e); diff --git a/tests/drivers/python/v5_8/transactions.py b/tests/drivers/python/v5_8/transactions.py index 58f05eeb1..9c8dd2f5f 100644 --- a/tests/drivers/python/v5_8/transactions.py +++ b/tests/drivers/python/v5_8/transactions.py @@ -14,7 +14,7 @@ import time -from neo4j import GraphDatabase, basic_auth +from neo4j import GraphDatabase from neo4j.exceptions import ClientError, TransientError @@ -75,11 +75,58 @@ def test_timeout(driver, set_timeout): raise Exception("The query should have timed out, but it didn't!") +def violate_constraint(tx): + tx.run("CREATE (n:Employee:Person {id: '123', alt_id: '100'});").consume() + + +def violate_constraint_on_intermediate_result(tx): + tx.run("CREATE (n:Employee:Person {id: '124', alt_id: '200'});").consume() + tx.run("MATCH (n {alt_id: '200'}) SET n.id = '123';").consume() # two (:Person {id: '123'}) + tx.run("MATCH (n {alt_id: '100'}) SET n.id = '122';").consume() # above violation fixed + + +def clear_db(session): + session.run("DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + session.run("DROP CONSTRAINT ON (n:Employee) ASSERT n.id IS UNIQUE;") + session.run("DROP CONSTRAINT ON (n:Employee) ASSERT EXISTS (n.id);") + + session.run("MATCH (n) DETACH DELETE n;") + + with GraphDatabase.driver("bolt://localhost:7687", auth=None, encrypted=False) as driver: + with driver.session() as session: + # Clear the DB + session.run("MATCH (n) DETACH DELETE n;") + + # Add constraints + session.run("CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT n.id IS UNIQUE;") + session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT EXISTS (n.id);") + + # Set the initial graph state + session.execute_write(lambda tx: tx.run("CREATE (n:Employee:Person {id: '123', alt_id: '100'}) RETURN n;")) + + # Run a transaction that violates a constraint + try: + session.execute_write(violate_constraint) + except TransientError: + pass + else: + clear_db(session) + raise Exception("neo4j.exceptions.TransientError should have been thrown!") + + # Run a transaction that violates no constraints even though an intermediate result does + try: + session.execute_write(violate_constraint_on_intermediate_result) + except TransientError: + clear_db(session) + raise Exception("neo4j.exceptions.TransientError should not have been thrown!") + + clear_db(session) def add_person(f, name, name2): with driver.session() as session: - session.write_transaction(f, name, name2) + session.execute_write(f, name, name2) # Wrong query. try: From e716c90031453fe92529219e8d59aa7cc2de255a Mon Sep 17 00:00:00 2001 From: andrejtonev <29177572+andrejtonev@users.noreply.github.com> Date: Mon, 4 Dec 2023 18:13:55 +0100 Subject: [PATCH 39/52] Fixed wrong handling of exceptions in SessionHL (#1560) --- src/glue/SessionHL.cpp | 52 +++++++++++++++++++++-- tests/drivers/python/v5_8/transactions.py | 16 +++---- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/glue/SessionHL.cpp b/src/glue/SessionHL.cpp index cc7910d1c..bff12d188 100644 --- a/src/glue/SessionHL.cpp +++ b/src/glue/SessionHL.cpp @@ -234,11 +234,55 @@ std::pair, std::optional> SessionHL::Interpret( throw memgraph::communication::bolt::ClientError(e.what()); } } -void SessionHL::RollbackTransaction() { interpreter_.RollbackTransaction(); } -void SessionHL::CommitTransaction() { interpreter_.CommitTransaction(); } -void SessionHL::BeginTransaction(const std::map &extra) { - interpreter_.BeginTransaction(ToQueryExtras(extra)); + +void SessionHL::RollbackTransaction() { + try { + interpreter_.RollbackTransaction(); + } catch (const memgraph::query::QueryException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + // Wrap QueryException into ClientError, because we want to allow the + // client to fix their query. + throw memgraph::communication::bolt::ClientError(e.what()); + } catch (const memgraph::query::ReplicationException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + throw memgraph::communication::bolt::ClientError(e.what()); + } } + +void SessionHL::CommitTransaction() { + try { + interpreter_.CommitTransaction(); + } catch (const memgraph::query::QueryException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + // Wrap QueryException into ClientError, because we want to allow the + // client to fix their query. + throw memgraph::communication::bolt::ClientError(e.what()); + } catch (const memgraph::query::ReplicationException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + throw memgraph::communication::bolt::ClientError(e.what()); + } +} + +void SessionHL::BeginTransaction(const std::map &extra) { + try { + interpreter_.BeginTransaction(ToQueryExtras(extra)); + } catch (const memgraph::query::QueryException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + // Wrap QueryException into ClientError, because we want to allow the + // client to fix their query. + throw memgraph::communication::bolt::ClientError(e.what()); + } catch (const memgraph::query::ReplicationException &e) { + // Count the number of specific exceptions thrown + metrics::IncrementCounter(GetExceptionName(e)); + throw memgraph::communication::bolt::ClientError(e.what()); + } +} + void SessionHL::Configure(const std::map &run_time_info) { #ifdef MG_ENTERPRISE std::string db; diff --git a/tests/drivers/python/v5_8/transactions.py b/tests/drivers/python/v5_8/transactions.py index 9c8dd2f5f..d26eef44c 100644 --- a/tests/drivers/python/v5_8/transactions.py +++ b/tests/drivers/python/v5_8/transactions.py @@ -96,12 +96,12 @@ def clear_db(session): with GraphDatabase.driver("bolt://localhost:7687", auth=None, encrypted=False) as driver: with driver.session() as session: # Clear the DB - session.run("MATCH (n) DETACH DELETE n;") + session.run("MATCH (n) DETACH DELETE n;").consume() # Add constraints - session.run("CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") - session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT n.id IS UNIQUE;") - session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT EXISTS (n.id);") + session.run("CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;").consume() + session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT n.id IS UNIQUE;").consume() + session.run("CREATE CONSTRAINT ON (n:Employee) ASSERT EXISTS (n.id);").consume() # Set the initial graph state session.execute_write(lambda tx: tx.run("CREATE (n:Employee:Person {id: '123', alt_id: '100'}) RETURN n;")) @@ -109,18 +109,18 @@ with GraphDatabase.driver("bolt://localhost:7687", auth=None, encrypted=False) a # Run a transaction that violates a constraint try: session.execute_write(violate_constraint) - except TransientError: + except ClientError: pass else: clear_db(session) - raise Exception("neo4j.exceptions.TransientError should have been thrown!") + raise Exception("neo4j.exceptions.ClientError should have been thrown!") # Run a transaction that violates no constraints even though an intermediate result does try: session.execute_write(violate_constraint_on_intermediate_result) - except TransientError: + except ClientError: clear_db(session) - raise Exception("neo4j.exceptions.TransientError should not have been thrown!") + raise Exception("neo4j.exceptions.ClientError should not have been thrown!") clear_db(session) From 74fa6d21f61b89a68f6d1daf3e1303e76faa1224 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:56:05 +0100 Subject: [PATCH 40/52] Implement parallel constraints recovery (#1545) --- config/flags.yaml | 4 + src/dbms/inmemory/replication_handlers.cpp | 18 ++- src/flags/general.cpp | 14 +- src/flags/general.hpp | 3 + src/memgraph.cpp | 4 +- src/storage/v2/config.hpp | 3 + .../v2/constraints/existence_constraints.cpp | 69 ++++++++- .../v2/constraints/existence_constraints.hpp | 44 +++--- src/storage/v2/constraints/utils.hpp | 42 +++++ .../v2/disk/edge_import_mode_cache.cpp | 9 +- .../v2/disk/edge_import_mode_cache.hpp | 5 +- src/storage/v2/disk/label_property_index.hpp | 4 - src/storage/v2/durability/durability.cpp | 144 +++++++++++------- src/storage/v2/durability/durability.hpp | 60 +++++--- src/storage/v2/durability/metadata.hpp | 4 +- src/storage/v2/durability/recovery_type.hpp | 23 +++ src/storage/v2/indices/indices_utils.hpp | 11 +- src/storage/v2/inmemory/label_index.cpp | 7 +- src/storage/v2/inmemory/label_index.hpp | 6 +- .../v2/inmemory/label_property_index.cpp | 8 +- .../v2/inmemory/label_property_index.hpp | 7 +- .../v2/inmemory/replication/recovery.cpp | 16 +- src/storage/v2/inmemory/storage.cpp | 30 ++-- src/storage/v2/inmemory/storage.hpp | 6 +- .../v2/inmemory/unique_constraints.cpp | 127 ++++++++++----- .../v2/inmemory/unique_constraints.hpp | 36 ++++- .../e2e/configuration/configuration_check.py | 2 - tests/e2e/configuration/default_config.py | 5 + tests/unit/storage_v2_durability_inmemory.cpp | 119 +++++++++++++++ 29 files changed, 628 insertions(+), 202 deletions(-) create mode 100644 src/storage/v2/constraints/utils.hpp create mode 100644 src/storage/v2/durability/recovery_type.hpp diff --git a/config/flags.yaml b/config/flags.yaml index cd3eee160..b551f90e4 100644 --- a/config/flags.yaml +++ b/config/flags.yaml @@ -111,6 +111,10 @@ modifications: value: "false" override: true + - name: "storage_parallel_schema_recovery" + value: "false" + override: true + - name: "storage_enable_schema_metadata" value: "false" override: true diff --git a/src/dbms/inmemory/replication_handlers.cpp b/src/dbms/inmemory/replication_handlers.cpp index a5f56ee3d..5eba61878 100644 --- a/src/dbms/inmemory/replication_handlers.cpp +++ b/src/dbms/inmemory/replication_handlers.cpp @@ -10,6 +10,7 @@ // licenses/APL.txt. #include "dbms/inmemory/replication_handlers.hpp" +#include #include "dbms/constants.hpp" #include "dbms/dbms_handler.hpp" #include "replication/replication_server.hpp" @@ -187,9 +188,9 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle storage::replication::Decoder decoder(req_reader); auto *storage = static_cast(db_acc->get()->storage()); - utils::EnsureDirOrDie(storage->snapshot_directory_); + utils::EnsureDirOrDie(storage->recovery_.snapshot_directory_); - const auto maybe_snapshot_path = decoder.ReadFile(storage->snapshot_directory_); + const auto maybe_snapshot_path = decoder.ReadFile(storage->recovery_.snapshot_directory_); MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!"); spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path); @@ -219,7 +220,10 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle storage->timestamp_ = std::max(storage->timestamp_, recovery_info.next_timestamp); spdlog::trace("Recovering indices and constraints from snapshot."); - storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_, + memgraph::storage::durability::RecoverIndicesAndStats(recovered_snapshot.indices_constraints.indices, + &storage->indices_, &storage->vertices_, + storage->name_id_mapper_.get()); + memgraph::storage::durability::RecoverConstraints(recovered_snapshot.indices_constraints.constraints, &storage->constraints_, &storage->vertices_, storage->name_id_mapper_.get()); } catch (const storage::durability::RecoveryFailure &e) { @@ -233,7 +237,7 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle spdlog::trace("Deleting old snapshot files due to snapshot recovery."); // Delete other durability files - auto snapshot_files = storage::durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); + auto snapshot_files = storage::durability::GetSnapshotFiles(storage->recovery_.snapshot_directory_, storage->uuid_); for (const auto &[path, uuid, _] : snapshot_files) { if (path != *maybe_snapshot_path) { spdlog::trace("Deleting snapshot file {}", path); @@ -242,7 +246,7 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle } spdlog::trace("Deleting old WAL files due to snapshot recovery."); - auto wal_files = storage::durability::GetWalFiles(storage->wal_directory_, storage->uuid_); + auto wal_files = storage::durability::GetWalFiles(storage->recovery_.wal_directory_, storage->uuid_); if (wal_files) { for (const auto &wal_file : *wal_files) { spdlog::trace("Deleting WAL file {}", wal_file.path); @@ -267,7 +271,7 @@ void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handle storage::replication::Decoder decoder(req_reader); auto *storage = static_cast(db_acc->get()->storage()); - utils::EnsureDirOrDie(storage->wal_directory_); + utils::EnsureDirOrDie(storage->recovery_.wal_directory_); for (auto i = 0; i < wal_file_number; ++i) { LoadWal(storage, &decoder); @@ -289,7 +293,7 @@ void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_hand storage::replication::Decoder decoder(req_reader); auto *storage = static_cast(db_acc->get()->storage()); - utils::EnsureDirOrDie(storage->wal_directory_); + utils::EnsureDirOrDie(storage->recovery_.wal_directory_); LoadWal(storage, &decoder); diff --git a/src/flags/general.cpp b/src/flags/general.cpp index 5a28bf16d..6bee2e5b3 100644 --- a/src/flags/general.cpp +++ b/src/flags/general.cpp @@ -104,9 +104,19 @@ DEFINE_bool(storage_snapshot_on_exit, false, "Controls whether the storage creat DEFINE_uint64(storage_items_per_batch, memgraph::storage::Config::Durability().items_per_batch, "The number of edges and vertices stored in a batch in a snapshot file."); +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,misc-unused-parameters) +DEFINE_VALIDATED_bool( + storage_parallel_index_recovery, false, + "Controls whether the index creation can be done in a multithreaded fashion.", { + spdlog::warn( + "storage_parallel_index_recovery flag is deprecated. Check storage_mode_parallel_schema_recovery for more " + "details."); + return true; + }); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -DEFINE_bool(storage_parallel_index_recovery, false, - "Controls whether the index creation can be done in a multithreaded fashion."); +DEFINE_bool(storage_parallel_schema_recovery, false, + "Controls whether the indices and constraints creation can be done in a multithreaded fashion."); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DEFINE_uint64(storage_recovery_thread_count, diff --git a/src/flags/general.hpp b/src/flags/general.hpp index eba59228b..890f32cd6 100644 --- a/src/flags/general.hpp +++ b/src/flags/general.hpp @@ -73,9 +73,12 @@ DECLARE_uint64(storage_wal_file_flush_every_n_tx); DECLARE_bool(storage_snapshot_on_exit); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_uint64(storage_items_per_batch); +// storage_parallel_index_recovery deprecated; use storage_parallel_schema_recovery instead // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_bool(storage_parallel_index_recovery); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +DECLARE_bool(storage_parallel_schema_recovery); +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_uint64(storage_recovery_thread_count); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DECLARE_bool(storage_enable_schema_metadata); diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 00a159aa5..057b30982 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -305,7 +305,9 @@ int main(int argc, char **argv) { .restore_replication_state_on_startup = FLAGS_replication_restore_state_on_startup, .items_per_batch = FLAGS_storage_items_per_batch, .recovery_thread_count = FLAGS_storage_recovery_thread_count, - .allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery}, + // deprecated + .allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery, + .allow_parallel_schema_creation = FLAGS_storage_parallel_schema_recovery}, .transaction = {.isolation_level = memgraph::flags::ParseIsolationLevel()}, .disk = {.main_storage_directory = FLAGS_data_directory + "/rocksdb_main_storage", .label_index_directory = FLAGS_data_directory + "/rocksdb_label_index", diff --git a/src/storage/v2/config.hpp b/src/storage/v2/config.hpp index 4f333b5d9..dee2afe87 100644 --- a/src/storage/v2/config.hpp +++ b/src/storage/v2/config.hpp @@ -65,7 +65,10 @@ struct Config { uint64_t items_per_batch{1'000'000}; uint64_t recovery_thread_count{8}; + // deprecated bool allow_parallel_index_creation{false}; + + bool allow_parallel_schema_creation{false}; friend bool operator==(const Durability &lrh, const Durability &rhs) = default; } durability; diff --git a/src/storage/v2/constraints/existence_constraints.cpp b/src/storage/v2/constraints/existence_constraints.cpp index a0d303c03..956e0a208 100644 --- a/src/storage/v2/constraints/existence_constraints.cpp +++ b/src/storage/v2/constraints/existence_constraints.cpp @@ -11,10 +11,11 @@ #include "storage/v2/constraints/existence_constraints.hpp" #include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/constraints/utils.hpp" #include "storage/v2/id_types.hpp" #include "storage/v2/mvcc.hpp" #include "utils/logging.hpp" - +#include "utils/rw_spin_lock.hpp" namespace memgraph::storage { bool ExistenceConstraints::ConstraintExists(LabelId label, PropertyId property) const { @@ -55,4 +56,70 @@ void ExistenceConstraints::LoadExistenceConstraints(const std::vector ExistenceConstraints::ValidateVertexOnConstraint( + const Vertex &vertex, const LabelId &label, const PropertyId &property) { + if (!vertex.deleted && utils::Contains(vertex.labels, label) && !vertex.properties.HasProperty(property)) { + return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set{property}}; + } + return std::nullopt; +} + +std::variant +ExistenceConstraints::GetCreationFunction( + const std::optional &par_exec_info) { + if (par_exec_info.has_value()) { + return ExistenceConstraints::MultipleThreadsConstraintValidation{par_exec_info.value()}; + } + return ExistenceConstraints::SingleThreadConstraintValidation{}; +} + +[[nodiscard]] std::optional ExistenceConstraints::ValidateVerticesOnConstraint( + utils::SkipList::Accessor vertices, LabelId label, PropertyId property, + const std::optional ¶llel_exec_info) { + auto calling_existence_validation_function = GetCreationFunction(parallel_exec_info); + return std::visit( + [&vertices, &label, &property](auto &calling_object) { return calling_object(vertices, label, property); }, + calling_existence_validation_function); +} + +std::optional ExistenceConstraints::MultipleThreadsConstraintValidation::operator()( + const utils::SkipList::Accessor &vertices, const LabelId &label, const PropertyId &property) { + utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; + + const auto &vertex_batches = parallel_exec_info.vertex_recovery_info; + MG_ASSERT(!vertex_batches.empty(), + "The size of batches should always be greater than zero if you want to use the parallel version of index " + "creation!"); + const auto thread_count = std::min(parallel_exec_info.thread_count, vertex_batches.size()); + + std::atomic batch_counter = 0; + memgraph::utils::Synchronized, utils::RWSpinLock> maybe_error{}; + { + std::vector threads; + threads.reserve(thread_count); + + for (auto i{0U}; i < thread_count; ++i) { + threads.emplace_back([&maybe_error, &vertex_batches, &batch_counter, &vertices, &label, &property]() { + do_per_thread_validation(maybe_error, ValidateVertexOnConstraint, vertex_batches, batch_counter, vertices, + label, property); + }); + } + } + if (maybe_error.Lock()->has_value()) { + return maybe_error->value(); + } + return std::nullopt; +} + +std::optional ExistenceConstraints::SingleThreadConstraintValidation::operator()( + const utils::SkipList::Accessor &vertices, const LabelId &label, const PropertyId &property) { + for (const Vertex &vertex : vertices) { + if (auto violation = ValidateVertexOnConstraint(vertex, label, property); violation.has_value()) { + return violation; + } + } + return std::nullopt; +} + } // namespace memgraph::storage diff --git a/src/storage/v2/constraints/existence_constraints.hpp b/src/storage/v2/constraints/existence_constraints.hpp index 77f7bc43a..c3b68828a 100644 --- a/src/storage/v2/constraints/existence_constraints.hpp +++ b/src/storage/v2/constraints/existence_constraints.hpp @@ -11,34 +11,45 @@ #pragma once +#include #include +#include +#include #include "storage/v2/constraints/constraint_violation.hpp" +#include "storage/v2/durability/recovery_type.hpp" #include "storage/v2/vertex.hpp" #include "utils/skip_list.hpp" +#include "utils/synchronized.hpp" namespace memgraph::storage { class ExistenceConstraints { + private: + std::vector> constraints_; + public: + struct MultipleThreadsConstraintValidation { + std::optional operator()(const utils::SkipList::Accessor &vertices, + const LabelId &label, const PropertyId &property); + + const durability::ParallelizedSchemaCreationInfo ¶llel_exec_info; + }; + struct SingleThreadConstraintValidation { + std::optional operator()(const utils::SkipList::Accessor &vertices, + const LabelId &label, const PropertyId &property); + }; + [[nodiscard]] static std::optional ValidateVertexOnConstraint(const Vertex &vertex, - LabelId label, - PropertyId property) { - if (!vertex.deleted && utils::Contains(vertex.labels, label) && !vertex.properties.HasProperty(property)) { - return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set{property}}; - } - return std::nullopt; - } + const LabelId &label, + const PropertyId &property); [[nodiscard]] static std::optional ValidateVerticesOnConstraint( - utils::SkipList::Accessor vertices, LabelId label, PropertyId property) { - for (const auto &vertex : vertices) { - if (auto violation = ValidateVertexOnConstraint(vertex, label, property); violation.has_value()) { - return violation; - } - } - return std::nullopt; - } + utils::SkipList::Accessor vertices, LabelId label, PropertyId property, + const std::optional ¶llel_exec_info = std::nullopt); + + static std::variant GetCreationFunction( + const std::optional &); bool ConstraintExists(LabelId label, PropertyId property) const; @@ -54,9 +65,6 @@ class ExistenceConstraints { std::vector> ListConstraints() const; void LoadExistenceConstraints(const std::vector &keys); - - private: - std::vector> constraints_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/constraints/utils.hpp b/src/storage/v2/constraints/utils.hpp new file mode 100644 index 000000000..ca48708ff --- /dev/null +++ b/src/storage/v2/constraints/utils.hpp @@ -0,0 +1,42 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include "storage/v2/vertex.hpp" +#include "utils/skip_list.hpp" + +namespace memgraph::storage { +template +void do_per_thread_validation(ErrorType &maybe_error, Func &&func, + const std::vector> &vertex_batches, + std::atomic &batch_counter, + const memgraph::utils::SkipList::Accessor &vertices, + Args &&...args) { + while (!maybe_error.ReadLock()->has_value()) { + const auto batch_index = batch_counter.fetch_add(1, std::memory_order_acquire); + if (batch_index >= vertex_batches.size()) { + return; + } + const auto &[gid_start, batch_size] = vertex_batches[batch_index]; + + auto vertex_curr = vertices.find(gid_start); + DMG_ASSERT(vertex_curr != vertices.end(), "No vertex was found with given gid"); + for (auto i{0U}; i < batch_size; ++i, ++vertex_curr) { + const auto violation = func(*vertex_curr, std::forward(args)...); + if (!violation.has_value()) [[likely]] { + continue; + } + maybe_error.WithLock([&violation](auto &maybe_error) { maybe_error = *violation; }); + break; + } + } +} +} // namespace memgraph::storage diff --git a/src/storage/v2/disk/edge_import_mode_cache.cpp b/src/storage/v2/disk/edge_import_mode_cache.cpp index cd1ca0dc2..b6621281f 100644 --- a/src/storage/v2/disk/edge_import_mode_cache.cpp +++ b/src/storage/v2/disk/edge_import_mode_cache.cpp @@ -43,8 +43,9 @@ InMemoryLabelPropertyIndex::Iterable EdgeImportModeCache::Vertices( storage, transaction); } -bool EdgeImportModeCache::CreateIndex(LabelId label, PropertyId property, - const std::optional ¶llel_exec_info) { +bool EdgeImportModeCache::CreateIndex( + LabelId label, PropertyId property, + const std::optional ¶llel_exec_info) { auto *mem_label_property_index = static_cast(in_memory_indices_.label_property_index_.get()); bool res = mem_label_property_index->CreateIndex(label, property, vertices_.access(), parallel_exec_info); @@ -54,8 +55,8 @@ bool EdgeImportModeCache::CreateIndex(LabelId label, PropertyId property, return res; } -bool EdgeImportModeCache::CreateIndex(LabelId label, - const std::optional ¶llel_exec_info) { +bool EdgeImportModeCache::CreateIndex( + LabelId label, const std::optional ¶llel_exec_info) { auto *mem_label_index = static_cast(in_memory_indices_.label_index_.get()); bool res = mem_label_index->CreateIndex(label, vertices_.access(), parallel_exec_info); if (res) { diff --git a/src/storage/v2/disk/edge_import_mode_cache.hpp b/src/storage/v2/disk/edge_import_mode_cache.hpp index 78ffcff59..02af960d5 100644 --- a/src/storage/v2/disk/edge_import_mode_cache.hpp +++ b/src/storage/v2/disk/edge_import_mode_cache.hpp @@ -42,9 +42,10 @@ class EdgeImportModeCache final { View view, Storage *storage, Transaction *transaction) const; bool CreateIndex(LabelId label, PropertyId property, - const std::optional ¶llel_exec_info = {}); + const std::optional ¶llel_exec_info = {}); - bool CreateIndex(LabelId label, const std::optional ¶llel_exec_info = {}); + bool CreateIndex(LabelId label, + const std::optional ¶llel_exec_info = {}); bool VerticesWithLabelPropertyScanned(LabelId label, PropertyId property) const; diff --git a/src/storage/v2/disk/label_property_index.hpp b/src/storage/v2/disk/label_property_index.hpp index 26f972d79..a6842200f 100644 --- a/src/storage/v2/disk/label_property_index.hpp +++ b/src/storage/v2/disk/label_property_index.hpp @@ -17,10 +17,6 @@ namespace memgraph::storage { -/// TODO: andi. Too many copies, extract at one place -using ParallelizedIndexCreationInfo = - std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; - class DiskLabelPropertyIndex : public storage::LabelPropertyIndex { public: explicit DiskLabelPropertyIndex(const Config &config); diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index 9a43a2876..6a89b7b5a 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -9,8 +9,6 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -#include "storage/v2/durability/durability.hpp" - #include #include #include @@ -20,23 +18,29 @@ #include #include +#include #include #include #include +#include "flags/all.hpp" +#include "gflags/gflags.h" #include "replication/epoch.hpp" +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/durability/metadata.hpp" #include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/wal.hpp" #include "storage/v2/inmemory/label_index.hpp" #include "storage/v2/inmemory/label_property_index.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" +#include "storage/v2/name_id_mapper.hpp" #include "utils/event_histogram.hpp" +#include "utils/flag_validation.hpp" #include "utils/logging.hpp" #include "utils/memory_tracker.hpp" #include "utils/message.hpp" #include "utils/timer.hpp" - namespace memgraph::metrics { extern const Event SnapshotRecoveryLatency_us; } // namespace memgraph::metrics @@ -134,15 +138,23 @@ std::optional> GetWalFiles(const std::filesystem: // indices and constraints must be recovered after the data recovery is done // to ensure that the indices and constraints are consistent at the end of the // recovery process. -void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, - Constraints *constraints, utils::SkipList *vertices, - NameIdMapper *name_id_mapper, - const std::optional ¶llel_exec_info) { + +void RecoverConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &constraints_metadata, + Constraints *constraints, utils::SkipList *vertices, NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info) { + RecoverExistenceConstraints(constraints_metadata, constraints, vertices, name_id_mapper, parallel_exec_info); + RecoverUniqueConstraints(constraints_metadata, constraints, vertices, name_id_mapper, parallel_exec_info); +} + +void RecoverIndicesAndStats(const RecoveredIndicesAndConstraints::IndicesMetadata &indices_metadata, Indices *indices, + utils::SkipList *vertices, NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info) { spdlog::info("Recreating indices from metadata."); + // Recover label indices. - spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size()); + spdlog::info("Recreating {} label indices from metadata.", indices_metadata.label.size()); auto *mem_label_index = static_cast(indices->label_index_.get()); - for (const auto &item : indices_constraints.indices.label) { + for (const auto &item : indices_metadata.label) { if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) { throw RecoveryFailure("The label index must be created here!"); } @@ -151,9 +163,10 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Label indices are recreated."); spdlog::info("Recreating index statistics from metadata."); + // Recover label indices statistics. - spdlog::info("Recreating {} label index statistics from metadata.", indices_constraints.indices.label_stats.size()); - for (const auto &item : indices_constraints.indices.label_stats) { + spdlog::info("Recreating {} label index statistics from metadata.", indices_metadata.label_stats.size()); + for (const auto &item : indices_metadata.label_stats) { mem_label_index->SetIndexStats(item.first, item.second); spdlog::info("Statistics for index on :{} are recreated from metadata", name_id_mapper->IdToName(item.first.AsUint())); @@ -161,10 +174,9 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Label indices statistics are recreated."); // Recover label+property indices. - spdlog::info("Recreating {} label+property indices from metadata.", - indices_constraints.indices.label_property.size()); + spdlog::info("Recreating {} label+property indices from metadata.", indices_metadata.label_property.size()); auto *mem_label_property_index = static_cast(indices->label_property_index_.get()); - for (const auto &item : indices_constraints.indices.label_property) { + for (const auto &item : indices_metadata.label_property) { if (!mem_label_property_index->CreateIndex(item.first, item.second, vertices->access(), parallel_exec_info)) throw RecoveryFailure("The label+property index must be created here!"); spdlog::info("Index on :{}({}) is recreated from metadata", name_id_mapper->IdToName(item.first.AsUint()), @@ -174,8 +186,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ // Recover label+property indices statistics. spdlog::info("Recreating {} label+property indices statistics from metadata.", - indices_constraints.indices.label_property_stats.size()); - for (const auto &item : indices_constraints.indices.label_property_stats) { + indices_metadata.label_property_stats.size()); + for (const auto &item : indices_metadata.label_property_stats) { const auto label_id = item.first; const auto property_id = item.second.first; const auto &stats = item.second.second; @@ -188,14 +200,20 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Indices are recreated."); spdlog::info("Recreating constraints from metadata."); - // Recover existence constraints. - spdlog::info("Recreating {} existence constraints from metadata.", indices_constraints.constraints.existence.size()); - for (const auto &[label, property] : indices_constraints.constraints.existence) { +} + +void RecoverExistenceConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &constraints_metadata, + Constraints *constraints, utils::SkipList *vertices, + NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info) { + spdlog::info("Recreating {} existence constraints from metadata.", constraints_metadata.existence.size()); + for (const auto &[label, property] : constraints_metadata.existence) { if (constraints->existence_constraints_->ConstraintExists(label, property)) { throw RecoveryFailure("The existence constraint already exists!"); } - if (auto violation = ExistenceConstraints::ValidateVerticesOnConstraint(vertices->access(), label, property); + if (auto violation = + ExistenceConstraints::ValidateVerticesOnConstraint(vertices->access(), label, property, parallel_exec_info); violation.has_value()) { throw RecoveryFailure("The existence constraint failed because it couldn't be validated!"); } @@ -205,38 +223,57 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ name_id_mapper->IdToName(property.AsUint())); } spdlog::info("Existence constraints are recreated from metadata."); +} - // Recover unique constraints. - spdlog::info("Recreating {} unique constraints from metadata.", indices_constraints.constraints.unique.size()); - for (const auto &item : indices_constraints.constraints.unique) { +void RecoverUniqueConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &constraints_metadata, + Constraints *constraints, utils::SkipList *vertices, NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info) { + spdlog::info("Recreating {} unique constraints from metadata.", constraints_metadata.unique.size()); + + for (const auto &[label, properties] : constraints_metadata.unique) { auto *mem_unique_constraints = static_cast(constraints->unique_constraints_.get()); - auto ret = mem_unique_constraints->CreateConstraint(item.first, item.second, vertices->access()); + auto ret = mem_unique_constraints->CreateConstraint(label, properties, vertices->access(), parallel_exec_info); if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) throw RecoveryFailure("The unique constraint must be created here!"); std::vector property_names; - property_names.reserve(item.second.size()); - for (const auto &prop : item.second) { + property_names.reserve(properties.size()); + for (const auto &prop : properties) { property_names.emplace_back(name_id_mapper->IdToName(prop.AsUint())); } const auto property_names_joined = utils::Join(property_names, ","); - spdlog::info("Unique constraint on :{}({}) is recreated from metadata", - name_id_mapper->IdToName(item.first.AsUint()), property_names_joined); + spdlog::info("Unique constraint on :{}({}) is recreated from metadata", name_id_mapper->IdToName(label.AsUint()), + property_names_joined); } spdlog::info("Unique constraints are recreated from metadata."); spdlog::info("Constraints are recreated from metadata."); } -std::optional RecoverData(const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, std::string *uuid, - ReplicationStorageState &repl_storage_state, utils::SkipList *vertices, - utils::SkipList *edges, std::atomic *edge_count, - NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, - const Config &config, uint64_t *wal_seq_num) { +std::optional GetParallelExecInfo(const RecoveryInfo &recovery_info, + const Config &config) { + return config.durability.allow_parallel_schema_creation + ? std::make_optional(ParallelizedSchemaCreationInfo{recovery_info.vertex_batches, + config.durability.recovery_thread_count}) + : std::nullopt; +} + +std::optional GetParallelExecInfoIndices(const RecoveryInfo &recovery_info, + const Config &config) { + return config.durability.allow_parallel_schema_creation || config.durability.allow_parallel_index_creation + ? std::make_optional(ParallelizedSchemaCreationInfo{recovery_info.vertex_batches, + config.durability.recovery_thread_count}) + : std::nullopt; +} + +std::optional Recovery::RecoverData(std::string *uuid, ReplicationStorageState &repl_storage_state, + utils::SkipList *vertices, utils::SkipList *edges, + std::atomic *edge_count, NameIdMapper *name_id_mapper, + Indices *indices, Constraints *constraints, const Config &config, + uint64_t *wal_seq_num) { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory, - wal_directory); - if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) { + spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory_, + wal_directory_); + if (!utils::DirExists(snapshot_directory_) && !utils::DirExists(wal_directory_)) { spdlog::warn(utils::MessageWithLink("Snapshot or WAL directory don't exist, there is nothing to recover.", "https://memgr.ph/durability")); return std::nullopt; @@ -245,13 +282,13 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di auto *const epoch_history = &repl_storage_state.history; utils::Timer timer; - auto snapshot_files = GetSnapshotFiles(snapshot_directory); + auto snapshot_files = GetSnapshotFiles(snapshot_directory_); RecoveryInfo recovery_info; RecoveredIndicesAndConstraints indices_constraints; std::optional snapshot_timestamp; if (!snapshot_files.empty()) { - spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory); + spdlog::info("Try recovering from snapshot directory {}.", wal_directory_); // UUID used for durability is the UUID of the last snapshot file. *uuid = snapshot_files.back().uuid; @@ -281,18 +318,17 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di snapshot_timestamp = recovered_snapshot->snapshot_info.start_timestamp; repl_storage_state.epoch_.SetEpoch(std::move(recovered_snapshot->snapshot_info.epoch_id)); - if (!utils::DirExists(wal_directory)) { - const auto par_exec_info = config.durability.allow_parallel_index_creation - ? std::make_optional(std::make_pair(recovery_info.vertex_batches, - config.durability.recovery_thread_count)) - : std::nullopt; - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); + if (!utils::DirExists(wal_directory_)) { + RecoverIndicesAndStats(indices_constraints.indices, indices, vertices, name_id_mapper, + GetParallelExecInfoIndices(recovery_info, config)); + RecoverConstraints(indices_constraints.constraints, constraints, vertices, name_id_mapper, + GetParallelExecInfo(recovery_info, config)); return recovered_snapshot->recovery_info; } } else { - spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory); + spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory_); std::error_code error_code; - if (!utils::DirExists(wal_directory)) return std::nullopt; + if (!utils::DirExists(wal_directory_)) return std::nullopt; // We use this smaller struct that contains only a subset of information // necessary for the rest of the recovery function. // Also, the struct is sorted primarily on the path it contains. @@ -306,7 +342,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di auto operator<=>(const WalFileInfo &) const = default; }; std::vector wal_files; - for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { + for (const auto &item : std::filesystem::directory_iterator(wal_directory_, error_code)) { if (!item.is_regular_file()) continue; try { auto info = ReadWalInfo(item.path()); @@ -327,7 +363,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di repl_storage_state.epoch_.SetEpoch(std::move(wal_files.back().epoch_id)); } - auto maybe_wal_files = GetWalFiles(wal_directory, *uuid); + auto maybe_wal_files = GetWalFiles(wal_directory_, *uuid); if (!maybe_wal_files) { spdlog::warn( utils::MessageWithLink("Couldn't get WAL file info from the WAL directory.", "https://memgr.ph/durability")); @@ -413,12 +449,10 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di spdlog::info("All necessary WAL files are loaded successfully."); } - const auto par_exec_info = - config.durability.allow_parallel_index_creation && !recovery_info.vertex_batches.empty() - ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) - : std::nullopt; - - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); + RecoverIndicesAndStats(indices_constraints.indices, indices, vertices, name_id_mapper, + GetParallelExecInfoIndices(recovery_info, config)); + RecoverConstraints(indices_constraints.constraints, constraints, vertices, name_id_mapper, + GetParallelExecInfo(recovery_info, config)); memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us, std::chrono::duration_cast(timer.Elapsed()).count()); diff --git a/src/storage/v2/durability/durability.hpp b/src/storage/v2/durability/durability.hpp index 8bb1223c4..97e2c7efc 100644 --- a/src/storage/v2/durability/durability.hpp +++ b/src/storage/v2/durability/durability.hpp @@ -23,6 +23,7 @@ #include "storage/v2/config.hpp" #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/durability/metadata.hpp" +#include "storage/v2/durability/recovery_type.hpp" #include "storage/v2/durability/wal.hpp" #include "storage/v2/edge.hpp" #include "storage/v2/indices/indices.hpp" @@ -94,27 +95,50 @@ std::optional> GetWalFiles(const std::filesystem: std::string_view uuid = "", std::optional current_seq_num = {}); -using ParallelizedIndexCreationInfo = - std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; - -// Helper function used to recover all discovered indices and constraints. The -// indices and constraints must be recovered after the data recovery is done -// to ensure that the indices and constraints are consistent at the end of the +// Helper function used to recover all discovered indices. The +// indices must be recovered after the data recovery is done +// to ensure that the indices consistent at the end of the // recovery process. /// @throw RecoveryFailure -void RecoverIndicesAndConstraints( - const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints, - utils::SkipList *vertices, NameIdMapper *name_id_mapper, - const std::optional ¶llel_exec_info = std::nullopt); +void RecoverIndicesAndStats(const RecoveredIndicesAndConstraints::IndicesMetadata &indices_metadata, Indices *indices, + utils::SkipList *vertices, NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info = std::nullopt); -/// Recovers data either from a snapshot and/or WAL files. +// Helper function used to recover all discovered constraints. The +// constraints must be recovered after the data recovery is done +// to ensure that the constraints are consistent at the end of the +// recovery process. /// @throw RecoveryFailure -/// @throw std::bad_alloc -std::optional RecoverData(const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, std::string *uuid, - ReplicationStorageState &repl_storage_state, utils::SkipList *vertices, - utils::SkipList *edges, std::atomic *edge_count, - NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, - const Config &config, uint64_t *wal_seq_num); +void RecoverConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &constraints_metadata, + Constraints *constraints, utils::SkipList *vertices, NameIdMapper *name_id_mapper, + const std::optional ¶llel_exec_info = std::nullopt); + +std::optional GetParallelExecInfo(const RecoveryInfo &recovery_info, + const Config &config); + +std::optional GetParallelExecInfoIndices(const RecoveryInfo &recovery_info, + const Config &config); + +void RecoverExistenceConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &, Constraints *, + utils::SkipList *, NameIdMapper *, + const std::optional &); + +void RecoverUniqueConstraints(const RecoveredIndicesAndConstraints::ConstraintsMetadata &, Constraints *, + utils::SkipList *, NameIdMapper *, + const std::optional &); +struct Recovery { + public: + /// Recovers data either from a snapshot and/or WAL files. + /// @throw RecoveryFailure + /// @throw std::bad_alloc + std::optional RecoverData(std::string *uuid, ReplicationStorageState &repl_storage_state, + utils::SkipList *vertices, utils::SkipList *edges, + std::atomic *edge_count, NameIdMapper *name_id_mapper, + Indices *indices, Constraints *constraints, const Config &config, + uint64_t *wal_seq_num); + + const std::filesystem::path snapshot_directory_; + const std::filesystem::path wal_directory_; +}; } // namespace memgraph::storage::durability diff --git a/src/storage/v2/durability/metadata.hpp b/src/storage/v2/durability/metadata.hpp index 1045d4f97..42e24e723 100644 --- a/src/storage/v2/durability/metadata.hpp +++ b/src/storage/v2/durability/metadata.hpp @@ -38,14 +38,14 @@ struct RecoveryInfo { /// Structure used to track indices and constraints during recovery. struct RecoveredIndicesAndConstraints { - struct { + struct IndicesMetadata { std::vector label; std::vector> label_property; std::vector> label_stats; std::vector>> label_property_stats; } indices; - struct { + struct ConstraintsMetadata { std::vector> existence; std::vector>> unique; } constraints; diff --git a/src/storage/v2/durability/recovery_type.hpp b/src/storage/v2/durability/recovery_type.hpp new file mode 100644 index 000000000..972cd53f2 --- /dev/null +++ b/src/storage/v2/durability/recovery_type.hpp @@ -0,0 +1,23 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include "storage/v2/id_types.hpp" + +namespace memgraph::storage::durability { +struct ParallelizedSchemaCreationInfo { + std::vector> vertex_recovery_info; + uint64_t thread_count; +}; +} // namespace memgraph::storage::durability diff --git a/src/storage/v2/indices/indices_utils.hpp b/src/storage/v2/indices/indices_utils.hpp index 0caad6686..59b492ba3 100644 --- a/src/storage/v2/indices/indices_utils.hpp +++ b/src/storage/v2/indices/indices_utils.hpp @@ -11,6 +11,7 @@ #include #include "storage/v2/delta.hpp" +#include "storage/v2/durability/recovery_type.hpp" #include "storage/v2/mvcc.hpp" #include "storage/v2/transaction.hpp" #include "storage/v2/vertex.hpp" @@ -20,9 +21,6 @@ namespace memgraph::storage { -using ParallelizedIndexCreationInfo = - std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; - /// Traverses deltas visible from transaction with start timestamp greater than /// the provided timestamp, and calls the provided callback function for each /// delta. If the callback ever returns true, traversal is stopped and the @@ -259,11 +257,12 @@ inline void CreateIndexOnSingleThread(utils::SkipList::Accessor &vertice template inline void CreateIndexOnMultipleThreads(utils::SkipList::Accessor &vertices, TSKiplistIter skiplist_iter, TIndex &index, TIndexKey key, - const ParallelizedIndexCreationInfo ¶llel_exec_info, const TFunc &func) { + const durability::ParallelizedSchemaCreationInfo ¶llel_exec_info, + const TFunc &func) { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - const auto &vertex_batches = parallel_exec_info.first; - const auto thread_count = std::min(parallel_exec_info.second, vertex_batches.size()); + const auto &vertex_batches = parallel_exec_info.vertex_recovery_info; + const auto thread_count = std::min(parallel_exec_info.thread_count, vertex_batches.size()); MG_ASSERT(!vertex_batches.empty(), "The size of batches should always be greater than zero if you want to use the parallel version of index " diff --git a/src/storage/v2/inmemory/label_index.cpp b/src/storage/v2/inmemory/label_index.cpp index 82ead4b44..b833c97ff 100644 --- a/src/storage/v2/inmemory/label_index.cpp +++ b/src/storage/v2/inmemory/label_index.cpp @@ -26,8 +26,9 @@ void InMemoryLabelIndex::UpdateOnAddLabel(LabelId added_label, Vertex *vertex_af acc.insert(Entry{vertex_after_update, tx.start_timestamp}); } -bool InMemoryLabelIndex::CreateIndex(LabelId label, utils::SkipList::Accessor vertices, - const std::optional ¶llel_exec_info) { +bool InMemoryLabelIndex::CreateIndex( + LabelId label, utils::SkipList::Accessor vertices, + const std::optional ¶llel_exec_info) { const auto create_index_seq = [this](LabelId label, utils::SkipList::Accessor &vertices, std::map>::iterator it) { using IndexAccessor = decltype(it->second.access()); @@ -42,7 +43,7 @@ bool InMemoryLabelIndex::CreateIndex(LabelId label, utils::SkipList::Acc const auto create_index_par = [this](LabelId label, utils::SkipList::Accessor &vertices, std::map>::iterator label_it, - const ParallelizedIndexCreationInfo ¶llel_exec_info) { + const durability::ParallelizedSchemaCreationInfo ¶llel_exec_info) { using IndexAccessor = decltype(label_it->second.access()); CreateIndexOnMultipleThreads(vertices, label_it, index_, label, parallel_exec_info, diff --git a/src/storage/v2/inmemory/label_index.hpp b/src/storage/v2/inmemory/label_index.hpp index 21df32deb..2411f0ba1 100644 --- a/src/storage/v2/inmemory/label_index.hpp +++ b/src/storage/v2/inmemory/label_index.hpp @@ -14,6 +14,7 @@ #include #include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/durability/recovery_type.hpp" #include "storage/v2/indices/label_index.hpp" #include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/vertex.hpp" @@ -22,9 +23,6 @@ namespace memgraph::storage { -using ParallelizedIndexCreationInfo = - std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; - class InMemoryLabelIndex : public storage::LabelIndex { private: struct Entry { @@ -47,7 +45,7 @@ class InMemoryLabelIndex : public storage::LabelIndex { /// @throw std::bad_alloc bool CreateIndex(LabelId label, utils::SkipList::Accessor vertices, - const std::optional ¶llel_exec_info); + const std::optional ¶llel_exec_info); /// Returns false if there was no index to drop bool DropIndex(LabelId label) override; diff --git a/src/storage/v2/inmemory/label_property_index.cpp b/src/storage/v2/inmemory/label_property_index.cpp index f61b9dd11..c8333fb95 100644 --- a/src/storage/v2/inmemory/label_property_index.cpp +++ b/src/storage/v2/inmemory/label_property_index.cpp @@ -35,9 +35,9 @@ bool InMemoryLabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) cons bool InMemoryLabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) const { return value == rhs; } -bool InMemoryLabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, - utils::SkipList::Accessor vertices, - const std::optional ¶llel_exec_info) { +bool InMemoryLabelPropertyIndex::CreateIndex( + LabelId label, PropertyId property, utils::SkipList::Accessor vertices, + const std::optional ¶llel_exec_info) { spdlog::trace("Vertices size when creating index: {}", vertices.size()); auto create_index_seq = [this](LabelId label, PropertyId property, utils::SkipList::Accessor &vertices, std::map, utils::SkipList>::iterator it) { @@ -54,7 +54,7 @@ bool InMemoryLabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, auto create_index_par = [this](LabelId label, PropertyId property, utils::SkipList::Accessor &vertices, std::map, utils::SkipList>::iterator label_property_it, - const ParallelizedIndexCreationInfo ¶llel_exec_info) { + const durability::ParallelizedSchemaCreationInfo ¶llel_exec_info) { using IndexAccessor = decltype(label_property_it->second.access()); CreateIndexOnMultipleThreads( diff --git a/src/storage/v2/inmemory/label_property_index.hpp b/src/storage/v2/inmemory/label_property_index.hpp index ae96a37f8..8bc4148bb 100644 --- a/src/storage/v2/inmemory/label_property_index.hpp +++ b/src/storage/v2/inmemory/label_property_index.hpp @@ -14,6 +14,7 @@ #include #include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/durability/recovery_type.hpp" #include "storage/v2/id_types.hpp" #include "storage/v2/indices/label_property_index.hpp" #include "storage/v2/indices/label_property_index_stats.hpp" @@ -23,10 +24,6 @@ namespace memgraph::storage { -/// TODO: andi. Too many copies, extract at one place -using ParallelizedIndexCreationInfo = - std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; - class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { private: struct Entry { @@ -46,7 +43,7 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { /// @throw std::bad_alloc bool CreateIndex(LabelId label, PropertyId property, utils::SkipList::Accessor vertices, - const std::optional ¶llel_exec_info); + const std::optional ¶llel_exec_info); /// @throw std::bad_alloc void UpdateOnAddLabel(LabelId added_label, Vertex *vertex_after_update, const Transaction &tx) override; diff --git a/src/storage/v2/inmemory/replication/recovery.cpp b/src/storage/v2/inmemory/replication/recovery.cpp index b62fdc2f9..536c7c8fc 100644 --- a/src/storage/v2/inmemory/replication/recovery.cpp +++ b/src/storage/v2/inmemory/replication/recovery.cpp @@ -133,8 +133,8 @@ std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileR std::optional current_wal_from_timestamp; std::unique_lock transaction_guard( - storage->engine_lock_); // Hold the storage lock so the current wal file cannot be changed - (void)locker_acc.AddPath(storage->wal_directory_); // Protect all WALs from being deleted + storage->engine_lock_); // Hold the storage lock so the current wal file cannot be changed + (void)locker_acc.AddPath(storage->recovery_.wal_directory_); // Protect all WALs from being deleted if (storage->wal_file_) { current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); @@ -146,20 +146,22 @@ std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileR // Read in finalized WAL files (excluding the current/active WAL) utils::OnScopeExit release_wal_dir( // Each individually used file will be locked, so at the end, the dir can be released - [&locker_acc, &wal_dir = storage->wal_directory_]() { (void)locker_acc.RemovePath(wal_dir); }); + [&locker_acc, &wal_dir = storage->recovery_.wal_directory_]() { (void)locker_acc.RemovePath(wal_dir); }); // Get WAL files, ordered by timestamp, from oldest to newest - auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); + auto wal_files = durability::GetWalFiles(storage->recovery_.wal_directory_, storage->uuid_, current_wal_seq_num); MG_ASSERT(wal_files, "Wal files could not be loaded"); if (transaction_guard.owns_lock()) transaction_guard.unlock(); // In case we didn't have a current wal file, we can unlock only now since there is no // guarantee what we'll see after we add the wal file // Read in snapshot files - (void)locker_acc.AddPath(storage->snapshot_directory_); // Protect all snapshots from being deleted + (void)locker_acc.AddPath(storage->recovery_.snapshot_directory_); // Protect all snapshots from being deleted utils::OnScopeExit release_snapshot_dir( // Each individually used file will be locked, so at the end, the dir can be released - [&locker_acc, &snapshot_dir = storage->snapshot_directory_]() { (void)locker_acc.RemovePath(snapshot_dir); }); - auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); + [&locker_acc, &snapshot_dir = storage->recovery_.snapshot_directory_]() { + (void)locker_acc.RemovePath(snapshot_dir); + }); + auto snapshot_files = durability::GetSnapshotFiles(storage->recovery_.snapshot_directory_, storage->uuid_); std::optional latest_snapshot{}; if (!snapshot_files.empty()) { latest_snapshot.emplace(std::move(snapshot_files.back())); diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 5b7b676b4..08aa896bf 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -12,6 +12,7 @@ #include "storage/v2/inmemory/storage.hpp" #include #include +#include #include "dbms/constants.hpp" #include "memory/global_memory_control.hpp" #include "storage/v2/durability/durability.hpp" @@ -65,9 +66,9 @@ using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) : Storage(config, storage_mode), - snapshot_directory_(config.durability.storage_directory / durability::kSnapshotDirectory), + recovery_{config.durability.storage_directory / durability::kSnapshotDirectory, + config.durability.storage_directory / durability::kWalDirectory}, lock_file_path_(config.durability.storage_directory / durability::kLockFile), - wal_directory_(config.durability.storage_directory / durability::kWalDirectory), uuid_(utils::GenerateUUID()), global_locker_(file_retainer_.AddLocker()) { MG_ASSERT(storage_mode != StorageMode::ON_DISK_TRANSACTIONAL, @@ -78,9 +79,9 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) // permission errors. This is done early to crash the database on startup // instead of crashing the database for the first time during runtime (which // could be an unpleasant surprise). - utils::EnsureDirOrDie(snapshot_directory_); + utils::EnsureDirOrDie(recovery_.snapshot_directory_); // Same reasoning as above. - utils::EnsureDirOrDie(wal_directory_); + utils::EnsureDirOrDie(recovery_.wal_directory_); // Verify that the user that started the process is the same user that is // the owner of the storage directory. @@ -98,9 +99,8 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) config_.durability.storage_directory); } if (config_.durability.recover_on_startup) { - auto info = - durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, repl_storage_state_, &vertices_, &edges_, - &edge_count_, name_id_mapper_.get(), &indices_, &constraints_, config_, &wal_seq_num_); + auto info = recovery_.RecoverData(&uuid_, repl_storage_state_, &vertices_, &edges_, &edge_count_, + name_id_mapper_.get(), &indices_, &constraints_, config_, &wal_seq_num_); if (info) { vertex_id_ = info->next_vertex_id; edge_id_ = info->next_edge_id; @@ -114,8 +114,8 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) bool files_moved = false; auto backup_root = config_.durability.storage_directory / durability::kBackupDirectory; for (const auto &[path, dirname, what] : - {std::make_tuple(snapshot_directory_, durability::kSnapshotDirectory, "snapshot"), - std::make_tuple(wal_directory_, durability::kWalDirectory, "WAL")}) { + {std::make_tuple(recovery_.snapshot_directory_, durability::kSnapshotDirectory, "snapshot"), + std::make_tuple(recovery_.wal_directory_, durability::kWalDirectory, "WAL")}) { if (!utils::DirExists(path)) continue; auto backup_curr = backup_root / dirname; std::error_code error_code; @@ -1200,8 +1200,8 @@ InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, Prop if (existence_constraints->ConstraintExists(label, property)) { return StorageExistenceConstraintDefinitionError{ConstraintDefinitionError{}}; } - if (auto violation = - ExistenceConstraints::ValidateVerticesOnConstraint(in_memory->vertices_.access(), label, property); + if (auto violation = ExistenceConstraints::ValidateVerticesOnConstraint(in_memory->vertices_.access(), label, + property, std::nullopt); violation.has_value()) { return StorageExistenceConstraintDefinitionError{violation.value()}; } @@ -1228,7 +1228,7 @@ InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const s auto *in_memory = static_cast(storage_); auto *mem_unique_constraints = static_cast(in_memory->constraints_.unique_constraints_.get()); - auto ret = mem_unique_constraints->CreateConstraint(label, properties, in_memory->vertices_.access()); + auto ret = mem_unique_constraints->CreateConstraint(label, properties, in_memory->vertices_.access(), std::nullopt); if (ret.HasError()) { return StorageUniqueConstraintDefinitionError{ret.GetError()}; } @@ -1707,7 +1707,7 @@ bool InMemoryStorage::InitializeWalFile(memgraph::replication::ReplicationEpoch if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL) return false; if (!wal_file_) { - wal_file_.emplace(wal_directory_, uuid_, epoch.id(), config_.items, name_id_mapper_.get(), wal_seq_num_++, + wal_file_.emplace(recovery_.wal_directory_, uuid_, epoch.id(), config_.items, name_id_mapper_.get(), wal_seq_num_++, &file_retainer_); } return true; @@ -2017,8 +2017,8 @@ utils::BasicResult InMemoryStorage::Create auto snapshot_creator = [this, &epoch]() { utils::Timer timer; auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION, storage_mode_); - durability::CreateSnapshot(this, &transaction, snapshot_directory_, wal_directory_, &vertices_, &edges_, uuid_, - epoch, repl_storage_state_.history, &file_retainer_); + durability::CreateSnapshot(this, &transaction, recovery_.snapshot_directory_, recovery_.wal_directory_, &vertices_, + &edges_, uuid_, epoch, repl_storage_state_.history, &file_retainer_); // Finalize snapshot transaction. commit_log_->MarkFinished(transaction.start_timestamp); diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 48d6f0cb7..2d2837467 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -345,6 +345,8 @@ class InMemoryStorage final : public Storage { void SetStorageMode(StorageMode storage_mode); + const durability::Recovery &GetRecovery() const noexcept { return recovery_; } + private: /// The force parameter determines the behaviour of the garbage collector. /// If it's set to true, it will behave as a global operation, i.e. it can't @@ -397,10 +399,10 @@ class InMemoryStorage final : public Storage { utils::SkipList edges_; // Durability - std::filesystem::path snapshot_directory_; + durability::Recovery recovery_; + std::filesystem::path lock_file_path_; utils::OutputFile lock_file_handle_; - std::filesystem::path wal_directory_; utils::Scheduler snapshot_runner_; utils::SpinLock snapshot_lock_; diff --git a/src/storage/v2/inmemory/unique_constraints.cpp b/src/storage/v2/inmemory/unique_constraints.cpp index 6a2945883..76cda1730 100644 --- a/src/storage/v2/inmemory/unique_constraints.cpp +++ b/src/storage/v2/inmemory/unique_constraints.cpp @@ -10,7 +10,13 @@ // licenses/APL.txt. #include "storage/v2/inmemory/unique_constraints.hpp" - +#include +#include "storage/v2/constraints/constraint_violation.hpp" +#include "storage/v2/constraints/utils.hpp" +#include "storage/v2/durability/recovery_type.hpp" +#include "storage/v2/id_types.hpp" +#include "utils/logging.hpp" +#include "utils/skip_list.hpp" namespace memgraph::storage { namespace { @@ -274,6 +280,75 @@ void InMemoryUniqueConstraints::UpdateBeforeCommit(const Vertex *vertex, const T } } +std::variant +InMemoryUniqueConstraints::GetCreationFunction( + const std::optional &par_exec_info) { + if (par_exec_info.has_value()) { + return InMemoryUniqueConstraints::MultipleThreadsConstraintValidation{par_exec_info.value()}; + } + return InMemoryUniqueConstraints::SingleThreadConstraintValidation{}; +} + +bool InMemoryUniqueConstraints::MultipleThreadsConstraintValidation::operator()( + const utils::SkipList::Accessor &vertex_accessor, utils::SkipList::Accessor &constraint_accessor, + const LabelId &label, const std::set &properties) { + utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; + const auto &vertex_batches = parallel_exec_info.vertex_recovery_info; + MG_ASSERT(!vertex_batches.empty(), + "The size of batches should always be greater than zero if you want to use the parallel version of index " + "creation!"); + const auto thread_count = std::min(parallel_exec_info.thread_count, vertex_batches.size()); + + std::atomic batch_counter = 0; + memgraph::utils::Synchronized, utils::RWSpinLock> has_error; + { + std::vector threads; + threads.reserve(thread_count); + for (auto i{0U}; i < thread_count; ++i) { + threads.emplace_back( + [&has_error, &vertex_batches, &batch_counter, &vertex_accessor, &constraint_accessor, &label, &properties]() { + do_per_thread_validation(has_error, DoValidate, vertex_batches, batch_counter, vertex_accessor, + constraint_accessor, label, properties); + }); + } + } + return has_error.Lock()->has_value(); +} + +bool InMemoryUniqueConstraints::SingleThreadConstraintValidation::operator()( + const utils::SkipList::Accessor &vertex_accessor, utils::SkipList::Accessor &constraint_accessor, + const LabelId &label, const std::set &properties) { + for (const Vertex &vertex : vertex_accessor) { + if (const auto violation = DoValidate(vertex, constraint_accessor, label, properties); violation.has_value()) { + return true; + } + } + return false; +} + +std::optional InMemoryUniqueConstraints::DoValidate( + const Vertex &vertex, utils::SkipList::Accessor &constraint_accessor, const LabelId &label, + const std::set &properties) { + if (vertex.deleted || !utils::Contains(vertex.labels, label)) { + return std::nullopt; + } + auto values = vertex.properties.ExtractPropertyValues(properties); + if (!values) { + return std::nullopt; + } + + // Check whether there already is a vertex with the same values for the + // given label and property. + auto it = constraint_accessor.find_equal_or_greater(*values); + if (it != constraint_accessor.end() && it->values == *values) { + return ConstraintViolation{ConstraintViolation::Type::UNIQUE, label, properties}; + } + + constraint_accessor.insert(Entry{std::move(*values), &vertex, 0}); + return std::nullopt; +} + void InMemoryUniqueConstraints::AbortEntries(std::span vertices, uint64_t exact_start_timestamp) { for (const auto &vertex : vertices) { for (const auto &label : vertex->labels) { @@ -297,8 +372,9 @@ void InMemoryUniqueConstraints::AbortEntries(std::span vert } utils::BasicResult -InMemoryUniqueConstraints::CreateConstraint(LabelId label, const std::set &properties, - utils::SkipList::Accessor vertices) { +InMemoryUniqueConstraints::CreateConstraint( + LabelId label, const std::set &properties, const utils::SkipList::Accessor &vertex_accessor, + const std::optional &par_exec_info) { if (properties.empty()) { return CreationStatus::EMPTY_PROPERTIES; } @@ -306,49 +382,28 @@ InMemoryUniqueConstraints::CreateConstraint(LabelId label, const std::set constraints_skip_list; + utils::SkipList::Accessor constraint_accessor{constraints_skip_list.access()}; - bool violation_found = false; + auto multi_single_thread_processing = GetCreationFunction(par_exec_info); - { - auto acc = constraint->second.access(); - - for (const Vertex &vertex : vertices) { - if (vertex.deleted || !utils::Contains(vertex.labels, label)) { - continue; - } - auto values = vertex.properties.ExtractPropertyValues(properties); - if (!values) { - continue; - } - - // Check whether there already is a vertex with the same values for the - // given label and property. - auto it = acc.find_equal_or_greater(*values); - if (it != acc.end() && it->values == *values) { - violation_found = true; - break; - } - - acc.insert(Entry{std::move(*values), &vertex, 0}); - } - } + bool violation_found = std::visit( + [&vertex_accessor, &constraint_accessor, &label, &properties](auto &multi_single_thread_processing) { + return multi_single_thread_processing(vertex_accessor, constraint_accessor, label, properties); + }, + multi_single_thread_processing); if (violation_found) { - // In the case of the violation, storage for the current constraint has to - // be removed. - constraints_.erase(constraint); return ConstraintViolation{ConstraintViolation::Type::UNIQUE, label, properties}; } + auto [it, _] = constraints_.emplace(std::make_pair(label, properties), std::move(constraints_skip_list)); + // Add the new constraint to the optimized structure only if there are no violations. - constraints_by_label_[label].insert({properties, &constraints_.at({label, properties})}); + constraints_by_label_[label].insert({properties, &it->second}); return CreationStatus::SUCCESS; } diff --git a/src/storage/v2/inmemory/unique_constraints.hpp b/src/storage/v2/inmemory/unique_constraints.hpp index d1e590357..15107f131 100644 --- a/src/storage/v2/inmemory/unique_constraints.hpp +++ b/src/storage/v2/inmemory/unique_constraints.hpp @@ -11,9 +11,17 @@ #pragma once +#include #include - +#include +#include +#include "storage/v2/constraints/constraint_violation.hpp" #include "storage/v2/constraints/unique_constraints.hpp" +#include "storage/v2/durability/recovery_type.hpp" +#include "storage/v2/id_types.hpp" +#include "utils/logging.hpp" +#include "utils/rw_spin_lock.hpp" +#include "utils/synchronized.hpp" namespace memgraph::storage { @@ -46,7 +54,24 @@ class InMemoryUniqueConstraints : public UniqueConstraints { bool operator==(const std::vector &rhs) const; }; + static std::optional DoValidate(const Vertex &vertex, + utils::SkipList::Accessor &constraint_accessor, + const LabelId &label, const std::set &properties); + public: + struct MultipleThreadsConstraintValidation { + bool operator()(const utils::SkipList::Accessor &vertex_accessor, + utils::SkipList::Accessor &constraint_accessor, const LabelId &label, + const std::set &properties); + + const durability::ParallelizedSchemaCreationInfo ¶llel_exec_info; + }; + struct SingleThreadConstraintValidation { + bool operator()(const utils::SkipList::Accessor &vertex_accessor, + utils::SkipList::Accessor &constraint_accessor, const LabelId &label, + const std::set &properties); + }; + /// Indexes the given vertex for relevant labels and properties. /// This method should be called before committing and validating vertices /// against unique constraints. @@ -67,9 +92,9 @@ class InMemoryUniqueConstraints : public UniqueConstraints { /// exceeds the maximum allowed number of properties, and /// `CreationStatus::SUCCESS` on success. /// @throw std::bad_alloc - utils::BasicResult CreateConstraint(LabelId label, - const std::set &properties, - utils::SkipList::Accessor vertices); + utils::BasicResult CreateConstraint( + LabelId label, const std::set &properties, const utils::SkipList::Accessor &vertex_accessor, + const std::optional &par_exec_info); /// Deletes the specified constraint. Returns `DeletionStatus::NOT_FOUND` if /// there is not such constraint in the storage, @@ -101,6 +126,9 @@ class InMemoryUniqueConstraints : public UniqueConstraints { void Clear() override; + static std::variant GetCreationFunction( + const std::optional &); + private: std::map>, utils::SkipList> constraints_; std::map, utils::SkipList *>> constraints_by_label_; diff --git a/tests/e2e/configuration/configuration_check.py b/tests/e2e/configuration/configuration_check.py index 3cdc919f4..8684b24d6 100644 --- a/tests/e2e/configuration/configuration_check.py +++ b/tests/e2e/configuration/configuration_check.py @@ -31,7 +31,6 @@ def test_does_default_config_match(): use the DEFINE_HIDDEN_* macro instead of DEFINE_* to prevent SHOW CONFIG from returning it. """ - assert len(config) == len(default_config.startup_config_dict), define_msg for flag in config: @@ -46,7 +45,6 @@ def test_does_default_config_match(): ] if flag_name in machine_dependent_configurations: continue - # default_value assert default_config.startup_config_dict[flag_name][0] == flag[1] # current_value diff --git a/tests/e2e/configuration/default_config.py b/tests/e2e/configuration/default_config.py index 025ba4d0a..e1d42b443 100644 --- a/tests/e2e/configuration/default_config.py +++ b/tests/e2e/configuration/default_config.py @@ -115,6 +115,11 @@ startup_config_dict = { "false", "Controls whether the index creation can be done in a multithreaded fashion.", ), + "storage_parallel_schema_recovery": ( + "false", + "false", + "Controls whether the indices and constraints creation can be done in a multithreaded fashion.", + ), "storage_enable_schema_metadata": ( "false", "false", diff --git a/tests/unit/storage_v2_durability_inmemory.cpp b/tests/unit/storage_v2_durability_inmemory.cpp index 725db9283..8a6d26fd1 100644 --- a/tests/unit/storage_v2_durability_inmemory.cpp +++ b/tests/unit/storage_v2_durability_inmemory.cpp @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include #include #include #include @@ -19,13 +20,18 @@ #include #include #include +#include #include #include #include +#include #include "dbms/database.hpp" #include "replication/state.hpp" #include "storage/v2/config.hpp" +#include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/constraints/existence_constraints.hpp" +#include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/marker.hpp" #include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/snapshot.hpp" @@ -34,10 +40,13 @@ #include "storage/v2/edge_accessor.hpp" #include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/inmemory/unique_constraints.hpp" +#include "storage/v2/storage_mode.hpp" #include "storage/v2/vertex_accessor.hpp" #include "utils/file.hpp" #include "utils/logging.hpp" #include "utils/timer.hpp" +#include "utils/uuid.hpp" using testing::Contains; using testing::UnorderedElementsAre; @@ -2703,3 +2712,113 @@ TEST_P(DurabilityTest, SnapshotAndWalMixedUUID) { ASSERT_FALSE(acc->Commit().HasError()); } } + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(DurabilityTest, ParallelConstraintsRecovery) { + // Create snapshot. + { + memgraph::storage::Config config{ + .items = {.properties_on_edges = GetParam()}, + .durability = {.storage_directory = storage_directory, .snapshot_on_exit = true, .items_per_batch = 13}}; + memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)}; + memgraph::dbms::Database db{config, repl_state}; + CreateBaseDataset(db.storage(), GetParam()); + VerifyDataset(db.storage(), DatasetType::ONLY_BASE, GetParam()); + CreateExtendedDataset(db.storage()); + VerifyDataset(db.storage(), DatasetType::BASE_WITH_EXTENDED, GetParam()); + } + + ASSERT_EQ(GetSnapshotsList().size(), 1); + ASSERT_EQ(GetBackupSnapshotsList().size(), 0); + ASSERT_EQ(GetWalsList().size(), 0); + ASSERT_EQ(GetBackupWalsList().size(), 0); + + // Recover snapshot. + memgraph::storage::Config config{.items = {.properties_on_edges = GetParam()}, + .durability = {.storage_directory = storage_directory, + .recover_on_startup = true, + .snapshot_on_exit = false, + .items_per_batch = 13, + .allow_parallel_index_creation = true}}; + memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)}; + memgraph::dbms::Database db{config, repl_state}; + VerifyDataset(db.storage(), DatasetType::BASE_WITH_EXTENDED, GetParam()); + { + auto acc = db.storage()->Access(); + auto vertex = acc->CreateVertex(); + auto edge = acc->CreateEdge(&vertex, &vertex, db.storage()->NameToEdgeType("et")); + ASSERT_TRUE(edge.HasValue()); + ASSERT_FALSE(acc->Commit().HasError()); + } +} + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(DurabilityTest, ConstraintsRecoveryFunctionSetting) { + memgraph::storage::Config config{.items = {.properties_on_edges = GetParam()}, + .durability = {.storage_directory = storage_directory, + .recover_on_startup = true, + .snapshot_on_exit = false, + .items_per_batch = 13, + .allow_parallel_schema_creation = true}}; + // Create snapshot. + { + config.durability.recover_on_startup = false; + config.durability.snapshot_on_exit = true; + memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)}; + memgraph::dbms::Database db{config, repl_state}; + CreateBaseDataset(db.storage(), GetParam()); + VerifyDataset(db.storage(), DatasetType::ONLY_BASE, GetParam()); + CreateExtendedDataset(db.storage()); + VerifyDataset(db.storage(), DatasetType::BASE_WITH_EXTENDED, GetParam()); + } + + ASSERT_EQ(GetSnapshotsList().size(), 1); + ASSERT_EQ(GetBackupSnapshotsList().size(), 0); + ASSERT_EQ(GetWalsList().size(), 0); + ASSERT_EQ(GetBackupWalsList().size(), 0); + + config.durability.recover_on_startup = true; + config.durability.snapshot_on_exit = false; + memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)}; + memgraph::utils::SkipList vertices; + memgraph::utils::SkipList edges; + std::unique_ptr name_id_mapper = std::make_unique(); + std::atomic edge_count{0}; + uint64_t wal_seq_num{0}; + std::string uuid{memgraph::utils::GenerateUUID()}; + memgraph::storage::Indices indices{config, memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL}; + memgraph::storage::Constraints constraints{config, memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL}; + memgraph::storage::ReplicationStorageState repl_storage_state; + + memgraph::storage::durability::Recovery recovery{ + config.durability.storage_directory / memgraph::storage::durability::kSnapshotDirectory, + config.durability.storage_directory / memgraph::storage::durability::kWalDirectory}; + + // Recover snapshot. + const auto info = recovery.RecoverData(&uuid, repl_storage_state, &vertices, &edges, &edge_count, + name_id_mapper.get(), &indices, &constraints, config, &wal_seq_num); + + MG_ASSERT(info.has_value(), "Info doesn't have value present"); + const auto par_exec_info = memgraph::storage::durability::GetParallelExecInfo(*info, config); + + MG_ASSERT(par_exec_info.has_value(), "Parallel exec info should have value present"); + + // Unique constraint choose function + auto *mem_unique_constraints = + static_cast(constraints.unique_constraints_.get()); + auto variant_unique_constraint_creation_func = mem_unique_constraints->GetCreationFunction(par_exec_info); + + const auto *pval = std::get_if( + &variant_unique_constraint_creation_func); + MG_ASSERT(pval, "Chose wrong function for recovery of data"); + + // Existence constraint choose function + auto *mem_existence_constraint = + static_cast(constraints.existence_constraints_.get()); + auto variant_existence_constraint_creation_func = mem_existence_constraint->GetCreationFunction(par_exec_info); + + const auto *pval_existence = + std::get_if( + &variant_existence_constraint_creation_func); + MG_ASSERT(pval_existence, "Chose wrong type of function for recovery of existence constraint data"); +} From eceed274d983d6f135189d945c407b30b6e506a6 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Tue, 5 Dec 2023 13:44:06 +0100 Subject: [PATCH 41/52] Relax mg assert condition on dealloc (#1492) --- src/utils/memory.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/memory.cpp b/src/utils/memory.cpp index 8bfbf4220..d09f70fc3 100644 --- a/src/utils/memory.cpp +++ b/src/utils/memory.cpp @@ -331,8 +331,7 @@ void PoolResource::DoDeallocate(void *p, size_t bytes, size_t alignment) { return; } // Deallocate a regular block, first check if last_dealloc_pool_ is suitable. - MG_ASSERT(last_dealloc_pool_, "Failed deallocation"); - if (last_dealloc_pool_->GetBlockSize() == block_size) return last_dealloc_pool_->Deallocate(p); + if (last_dealloc_pool_ && last_dealloc_pool_->GetBlockSize() == block_size) return last_dealloc_pool_->Deallocate(p); // Find the pool with equal block_size. impl::Pool pool(block_size, max_blocks_per_chunk_, GetUpstreamResource()); auto it = std::lower_bound(pools_.begin(), pools_.end(), pool, From 7a9c4f5ec4e8f68746ec59aa7265df3b82539894 Mon Sep 17 00:00:00 2001 From: gvolfing <107616712+gvolfing@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:52:28 +0100 Subject: [PATCH 42/52] Fix logic in RelWithDebInfo mode (#1397) In and only in RelWithDebInfo mode the access of the maybe_unused variable results in segfaults, this change is making sure that does no happen ever if the maybe_unused variable is nullopt without changing the overall logic. --- .../frontend/ast/cypher_main_visitor.cpp | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 16a5f4828..7002ee4b9 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -1276,28 +1276,59 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur call_proc->result_identifiers_.push_back(storage_->Create(result_alias)); } } else { - const auto &maybe_found = - procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource()); - if (!maybe_found) { - throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); + call_proc->is_write_ = maybe_found->second->info.is_write; + + auto *yield_ctx = ctx->yieldProcedureResults(); + if (!yield_ctx) { + if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) { + throw SemanticException( + "CALL without YIELD may only be used on procedures which do not " + "return any result fields."); + } + // When we return, we will release the lock on modules. This means that + // someone may reload the procedure and change the result signature. But to + // keep the implementation simple, we ignore the case as the rest of the + // code doesn't really care whether we yield or not, so it should not break. + return call_proc; } - const auto &[module, proc] = *maybe_found; - call_proc->result_fields_.reserve(proc->results.size()); - call_proc->result_identifiers_.reserve(proc->results.size()); - for (const auto &[result_name, desc] : proc->results) { - bool is_deprecated = desc.second; - if (is_deprecated) continue; - call_proc->result_fields_.emplace_back(result_name); - call_proc->result_identifiers_.push_back(storage_->Create(std::string(result_name))); + if (yield_ctx->getTokens(MemgraphCypher::ASTERISK).empty()) { + call_proc->result_fields_.reserve(yield_ctx->procedureResult().size()); + call_proc->result_identifiers_.reserve(yield_ctx->procedureResult().size()); + for (auto *result : yield_ctx->procedureResult()) { + MG_ASSERT(result->variable().size() == 1 || result->variable().size() == 2); + call_proc->result_fields_.push_back(std::any_cast(result->variable()[0]->accept(this))); + std::string result_alias; + if (result->variable().size() == 2) { + result_alias = std::any_cast(result->variable()[1]->accept(this)); + } else { + result_alias = std::any_cast(result->variable()[0]->accept(this)); + } + call_proc->result_identifiers_.push_back(storage_->Create(result_alias)); + } + } else { + const auto &maybe_found = + procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource()); + if (!maybe_found) { + throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); + } + const auto &[module, proc] = *maybe_found; + call_proc->result_fields_.reserve(proc->results.size()); + call_proc->result_identifiers_.reserve(proc->results.size()); + for (const auto &[result_name, desc] : proc->results) { + bool is_deprecated = desc.second; + if (is_deprecated) continue; + call_proc->result_fields_.emplace_back(result_name); + call_proc->result_identifiers_.push_back(storage_->Create(std::string(result_name))); + } + // When we leave the scope, we will release the lock on modules. This means + // that someone may reload the procedure and change its result signature. We + // are fine with this, because if new result fields were added then we yield + // the subset of those and that will appear to a user as if they used the + // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the + // new fields as well. In case the result signature has had some result + // fields removed, then the query execution will report an error that we are + // yielding missing fields. The user can then just retry the query. } - // When we leave the scope, we will release the lock on modules. This means - // that someone may reload the procedure and change its result signature. We - // are fine with this, because if new result fields were added then we yield - // the subset of those and that will appear to a user as if they used the - // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the - // new fields as well. In case the result signature has had some result - // fields removed, then the query execution will report an error that we are - // yielding missing fields. The user can then just retry the query. } return call_proc; From e56e516f948572ac7d58d0e03b4a28a289bbcb67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:54:43 +0100 Subject: [PATCH 43/52] Add BigMemory label to the release build (#1568) --- .github/workflows/release_debian10.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_debian10.yaml b/.github/workflows/release_debian10.yaml index b7f4adc7d..9a38e4cfb 100644 --- a/.github/workflows/release_debian10.yaml +++ b/.github/workflows/release_debian10.yaml @@ -178,7 +178,7 @@ jobs: release_build: name: "Release build" - runs-on: [self-hosted, Linux, X64, Debian10] + runs-on: [self-hosted, Linux, X64, Debian10, BigMemory] env: THREADS: 24 MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} From 340057f95975f8bca186eb5d635baeae9878374d Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Fri, 8 Dec 2023 09:23:20 +0100 Subject: [PATCH 44/52] Add robustness on memory tracker stress test (#1394) --- tests/stress/memory_tracker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/stress/memory_tracker.py b/tests/stress/memory_tracker.py index ec67fe97b..d0f8bc350 100644 --- a/tests/stress/memory_tracker.py +++ b/tests/stress/memory_tracker.py @@ -260,10 +260,11 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None: # Problem with test using detach delete and memory tracker # is that memory tracker gets updated immediately # whereas RES takes some time - cnt_again = 3 + # Tries 10 times or fails + cnt_again = 10 skip_failure = False - # 10% is maximum increment, afterwards is fail - multiplier = 1 + # 10% is maximum diff for this test to pass + multiplier = 1.10 while cnt_again: new_memory_tracker, new_res_data = get_storage_data(session) @@ -277,7 +278,6 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None: f"RES data: {new_res_data}, multiplier: {multiplier}" ) break - multiplier += 0.05 cnt_again -= 1 if not skip_failure: log.info(memory_tracker, initial_diff, res_data) From 375c3c5dddccd2a9875e6047aecb305e4a719e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:22:53 +0100 Subject: [PATCH 45/52] Update BSL license change date (#1571) --- licenses/BSL.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/licenses/BSL.txt b/licenses/BSL.txt index deca9a407..9cca9dccf 100644 --- a/licenses/BSL.txt +++ b/licenses/BSL.txt @@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the 3. using the Licensed Work to create a work or solution which competes (or might reasonably be expected to compete) with the Licensed Work. -CHANGE DATE: 2027-30-10 +CHANGE DATE: 2027-08-12 CHANGE LICENSE: Apache License, Version 2.0 For information about alternative licensing arrangements, please visit: https://memgraph.com/legal. From 21bbc196ae9c4c38d4704dc9667768a259347310 Mon Sep 17 00:00:00 2001 From: Gareth Andrew Lloyd Date: Wed, 13 Dec 2023 13:19:01 +0000 Subject: [PATCH 46/52] Cleanup filesystem after unittest (#1581) --- tests/unit/dbms_handler.cpp | 1 + tests/unit/dbms_handler_community.cpp | 1 + tests/unit/interpreter.cpp | 2 ++ tests/unit/query_dump.cpp | 11 ++++++++++- tests/unit/storage_v2_isolation_level.cpp | 4 ++-- tests/unit/storage_v2_storage_mode.cpp | 2 ++ 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/unit/dbms_handler.cpp b/tests/unit/dbms_handler.cpp index a811e4159..0ea4197fb 100644 --- a/tests/unit/dbms_handler.cpp +++ b/tests/unit/dbms_handler.cpp @@ -58,6 +58,7 @@ class TestEnvironment : public ::testing::Environment { void TearDown() override { ptr_.reset(); auth.reset(); + std::filesystem::remove_all(storage_directory); } static std::unique_ptr ptr_; diff --git a/tests/unit/dbms_handler_community.cpp b/tests/unit/dbms_handler_community.cpp index 3848cd347..860f70ba0 100644 --- a/tests/unit/dbms_handler_community.cpp +++ b/tests/unit/dbms_handler_community.cpp @@ -58,6 +58,7 @@ class TestEnvironment : public ::testing::Environment { void TearDown() override { ptr_.reset(); auth.reset(); + std::filesystem::remove_all(storage_directory); } static std::unique_ptr ptr_; diff --git a/tests/unit/interpreter.cpp b/tests/unit/interpreter.cpp index 57bb79db8..bd587e7df 100644 --- a/tests/unit/interpreter.cpp +++ b/tests/unit/interpreter.cpp @@ -101,6 +101,8 @@ class InterpreterTest : public ::testing::Test { disk_test_utils::RemoveRocksDbDirs(testSuite); disk_test_utils::RemoveRocksDbDirs(testSuiteCsv); } + + std::filesystem::remove_all(data_directory); } InterpreterFaker default_interpreter{&interpreter_context, db}; diff --git a/tests/unit/query_dump.cpp b/tests/unit/query_dump.cpp index 5556ab90a..63019ad28 100644 --- a/tests/unit/query_dump.cpp +++ b/tests/unit/query_dump.cpp @@ -700,6 +700,11 @@ TYPED_TEST(DumpTest, CheckStateVertexWithMultipleProperties) { config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s1").disk; config.force_on_disk = true; } + auto on_exit_s1 = memgraph::utils::OnScopeExit{[&]() { + if constexpr (std::is_same_v) { + disk_test_utils::RemoveRocksDbDirs("query-dump-s1"); + } + }}; memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(config)); memgraph::utils::Gatekeeper db_gk(config, repl_state); @@ -814,7 +819,11 @@ TYPED_TEST(DumpTest, CheckStateSimpleGraph) { config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s2").disk; config.force_on_disk = true; } - + auto on_exit_s2 = memgraph::utils::OnScopeExit{[&]() { + if constexpr (std::is_same_v) { + disk_test_utils::RemoveRocksDbDirs("query-dump-s2"); + } + }}; memgraph::replication::ReplicationState repl_state{ReplicationStateRootPath(config)}; memgraph::utils::Gatekeeper db_gk{config, repl_state}; auto db_acc_opt = db_gk.access(); diff --git a/tests/unit/storage_v2_isolation_level.cpp b/tests/unit/storage_v2_isolation_level.cpp index 5cdfb5656..d2ae14d8f 100644 --- a/tests/unit/storage_v2_isolation_level.cpp +++ b/tests/unit/storage_v2_isolation_level.cpp @@ -15,6 +15,7 @@ #include "storage/v2/disk/storage.hpp" #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/isolation_level.hpp" +#include "utils/on_scope_exit.hpp" namespace { int64_t VerticesCount(memgraph::storage::Storage::Accessor *accessor) { @@ -113,6 +114,7 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) { for (const auto override_isolation_level : isolation_levels) { std::unique_ptr storage(new memgraph::storage::DiskStorage(config)); + auto on_exit = memgraph::utils::OnScopeExit{[&]() { disk_test_utils::RemoveRocksDbDirs(testSuite); }}; try { this->TestVisibility(storage, default_isolation_level, override_isolation_level); } catch (memgraph::utils::NotYetImplemented &) { @@ -120,10 +122,8 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) { override_isolation_level != memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION) { continue; } - disk_test_utils::RemoveRocksDbDirs(testSuite); throw; } - disk_test_utils::RemoveRocksDbDirs(testSuite); } } diff --git a/tests/unit/storage_v2_storage_mode.cpp b/tests/unit/storage_v2_storage_mode.cpp index 3daea2e69..49ee633c5 100644 --- a/tests/unit/storage_v2_storage_mode.cpp +++ b/tests/unit/storage_v2_storage_mode.cpp @@ -75,6 +75,8 @@ class StorageModeMultiTxTest : public ::testing::Test { return tmp; }(); // iile + void TearDown() override { std::filesystem::remove_all(data_directory); } + memgraph::storage::Config config{.durability.storage_directory = data_directory, .disk.main_storage_directory = data_directory / "disk"}; From b35df12c1ab1841ad412bee100c55a1bb63de215 Mon Sep 17 00:00:00 2001 From: Gareth Andrew Lloyd Date: Thu, 14 Dec 2023 13:36:33 +0000 Subject: [PATCH 47/52] Cleanup filesystem after e2e tests (#1584) --- tests/e2e/interactive_mg_runner.py | 33 ++++++++++++++++++------------ tests/e2e/memgraph.py | 15 +++++++++++--- tests/e2e/runner.py | 6 +++--- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/tests/e2e/interactive_mg_runner.py b/tests/e2e/interactive_mg_runner.py index 769516961..13aa951db 100644 --- a/tests/e2e/interactive_mg_runner.py +++ b/tests/e2e/interactive_mg_runner.py @@ -80,6 +80,8 @@ ACTIONS = { "quit": lambda _: sys.exit(1), } +CLEANUP_DIRECTORIES_ON_EXIT = False + log = logging.getLogger("memgraph.tests.e2e") @@ -109,10 +111,11 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_ assert not is_port_in_use( extract_bolt_port(args) ), "If this raises, you are trying to start an instance on a port already used by one already running instance." - mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl) - MEMGRAPH_INSTANCES[name] = mg_instance + log_file_path = os.path.join(BUILD_DIR, "logs", log_file) data_directory_path = os.path.join(BUILD_DIR, data_directory) + mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl, {data_directory_path}) + MEMGRAPH_INSTANCES[name] = mg_instance binary_args = args + ["--log-file", log_file_path] + ["--data-directory", data_directory_path] if len(procdir) != 0: @@ -122,39 +125,43 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_ assert mg_instance.is_running(), "An error occured after starting Memgraph instance: application stopped running." -def stop_all(): +def stop_all(keep_directories=True): for mg_instance in MEMGRAPH_INSTANCES.values(): - mg_instance.stop() + mg_instance.stop(keep_directories) MEMGRAPH_INSTANCES.clear() -def stop_instance(context, name): +def stop_instance(context, name, keep_directories=True): for key, _ in context.items(): if key != name: continue - MEMGRAPH_INSTANCES[name].stop() + MEMGRAPH_INSTANCES[name].stop(keep_directories) MEMGRAPH_INSTANCES.pop(name) -def stop(context, name): +def stop(context, name, keep_directories=True): if name != "all": - stop_instance(context, name) + stop_instance(context, name, keep_directories) return stop_all() -def kill(context, name): +def kill(context, name, keep_directories=True): for key in context.keys(): if key != name: continue - MEMGRAPH_INSTANCES[name].kill() + MEMGRAPH_INSTANCES[name].kill(keep_directories) MEMGRAPH_INSTANCES.pop(name) +def cleanup_directories_on_exit(value=True): + CLEANUP_DIRECTORIES_ON_EXIT = value + + @atexit.register def cleanup(): - stop_all() + stop_all(CLEANUP_DIRECTORIES_ON_EXIT) def start_instance(context, name, procdir): @@ -184,8 +191,8 @@ def start_instance(context, name, procdir): assert len(mg_instances) == 1 -def start_all(context, procdir=""): - stop_all() +def start_all(context, procdir="", keep_directories=True): + stop_all(keep_directories) for key, _ in context.items(): start_instance(context, key, procdir) diff --git a/tests/e2e/memgraph.py b/tests/e2e/memgraph.py index a65bed2ed..d5a62a388 100755 --- a/tests/e2e/memgraph.py +++ b/tests/e2e/memgraph.py @@ -11,6 +11,7 @@ import copy import os +import shutil import subprocess import sys import time @@ -56,13 +57,14 @@ def replace_paths(path): class MemgraphInstanceRunner: - def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False): + def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False, delete_on_stop=None): self.host = "127.0.0.1" self.bolt_port = None self.binary_path = binary_path self.args = None self.proc_mg = None self.ssl = use_ssl + self.delete_on_stop = delete_on_stop def execute_setup_queries(self, setup_queries): if setup_queries is None: @@ -128,7 +130,7 @@ class MemgraphInstanceRunner: return False return True - def stop(self): + def stop(self, keep_directories=False): if not self.is_running(): return @@ -140,9 +142,16 @@ class MemgraphInstanceRunner: time.sleep(1) - def kill(self): + if not keep_directories: + for folder in self.delete_on_stop or {}: + shutil.rmtree(folder) + + def kill(self, keep_directories=False): if not self.is_running(): return self.proc_mg.kill() code = self.proc_mg.wait() + if not keep_directories: + for folder in self.delete_on_stop or {}: + shutil.rmtree(folder) assert code == -9, "The killed Memgraph process exited with non-nine!" diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index f2ca5cce5..949670d43 100755 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -53,8 +53,8 @@ def run(args): # Setup. @atexit.register - def cleanup(): - interactive_mg_runner.stop_all() + def cleanup(keep_directories=True): + interactive_mg_runner.stop_all(keep_directories) if "pre_set_workload" in workload: binary = os.path.join(BUILD_DIR, workload["pre_set_workload"]) @@ -92,7 +92,7 @@ def run(args): data = mg_instance.query(validation["query"], conn)[0][0] assert data == validation["expected"] conn.close() - cleanup() + cleanup(keep_directories=False) log.info("%s PASSED.", workload_name) From 39ee248d34d3ddf1d836803835f3b0342c520e92 Mon Sep 17 00:00:00 2001 From: DavIvek Date: Mon, 18 Dec 2023 11:47:24 +0100 Subject: [PATCH 48/52] Fix java drivers test (#1577) --- tests/drivers/java/v5_8/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/drivers/java/v5_8/run.sh b/tests/drivers/java/v5_8/run.sh index 0e85f68df..03400e385 100755 --- a/tests/drivers/java/v5_8/run.sh +++ b/tests/drivers/java/v5_8/run.sh @@ -9,8 +9,8 @@ fi if [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64" fi -if [ -d "/opt/apache-maven-3.9.2" ]; then - export M2_HOME="/opt/apache-maven-3.9.2" +if [ -d "/opt/apache-maven-3.9.3" ]; then + export M2_HOME="/opt/apache-maven-3.9.3" fi export PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH" From cb4d4db81336269ac9c810283c1e171a9af15e6f Mon Sep 17 00:00:00 2001 From: DavIvek Date: Mon, 18 Dec 2023 14:34:21 +0100 Subject: [PATCH 49/52] Fix schema query module (#1510) --- query_modules/schema.cpp | 106 +++++++++++++++++++---- tests/e2e/query_modules/schema_test.py | 115 ++++++++++++++++++++++++- 2 files changed, 199 insertions(+), 22 deletions(-) diff --git a/query_modules/schema.cpp b/query_modules/schema.cpp index 1b3035bab..848ccedc4 100644 --- a/query_modules/schema.cpp +++ b/query_modules/schema.cpp @@ -108,31 +108,83 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t record.Insert(std::string(kReturnMandatory).c_str(), mandatory); } +struct Property { + std::string name; + mgp::Value value; + + Property(const std::string &name, mgp::Value &&value) : name(name), value(std::move(value)) {} +}; + +struct LabelsHash { + std::size_t operator()(const std::set &set) const { + std::size_t seed = set.size(); + for (const auto &i : set) { + seed ^= std::hash{}(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; + } +}; + +struct LabelsComparator { + bool operator()(const std::set &lhs, const std::set &rhs) const { return lhs == rhs; } +}; + +struct PropertyComparator { + bool operator()(const Property &lhs, const Property &rhs) const { return lhs.name < rhs.name; } +}; + +struct PropertyInfo { + std::set properties; + bool mandatory; +}; + void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; const auto record_factory = mgp::RecordFactory(result); try { - const mgp::Graph graph = mgp::Graph(memgraph_graph); - for (auto node : graph.Nodes()) { - std::string type; - mgp::List labels = mgp::List(); + std::unordered_map, PropertyInfo, LabelsHash, LabelsComparator> node_types_properties; + + for (auto node : mgp::Graph(memgraph_graph).Nodes()) { + std::set labels_set = {}; for (auto label : node.Labels()) { - labels.AppendExtend(mgp::Value(label)); - type += ":`" + std::string(label) + "`"; + labels_set.emplace(label); + } + + if (node_types_properties.find(labels_set) == node_types_properties.end()) { + node_types_properties[labels_set] = PropertyInfo{std::set(), true}; } if (node.Properties().empty()) { - auto record = record_factory.NewRecord(); - ProcessPropertiesNode(record, type, labels, "", "", false); + node_types_properties[labels_set].mandatory = false; // if there is node with no property, it is not mandatory continue; } + auto &property_info = node_types_properties.at(labels_set); for (auto &[key, prop] : node.Properties()) { - auto property_type = mgp::List(); + property_info.properties.emplace(key, std::move(prop)); + if (property_info.mandatory) { + property_info.mandatory = + property_info.properties.size() == 1; // if there is only one property, it is mandatory + } + } + } + + for (auto &[labels, property_info] : node_types_properties) { + std::string label_type; + mgp::List labels_list = mgp::List(); + for (auto const &label : labels) { + label_type += ":`" + std::string(label) + "`"; + labels_list.AppendExtend(mgp::Value(label)); + } + for (auto const &prop : property_info.properties) { auto record = record_factory.NewRecord(); - property_type.AppendExtend(mgp::Value(TypeOf(prop.Type()))); - ProcessPropertiesNode(record, type, labels, key, property_type, true); + ProcessPropertiesNode(record, label_type, labels_list, prop.name, TypeOf(prop.value.Type()), + property_info.mandatory); + } + if (property_info.properties.empty()) { + auto record = record_factory.NewRecord(); + ProcessPropertiesNode(record, label_type, labels_list, "", "", false); } } @@ -144,23 +196,41 @@ void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; + + std::unordered_map rel_types_properties; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); - for (auto rel : graph.Relationships()) { - std::string type = ":`" + std::string(rel.Type()) + "`"; + std::string rel_type = std::string(rel.Type()); + if (rel_types_properties.find(rel_type) == rel_types_properties.end()) { + rel_types_properties[rel_type] = PropertyInfo{std::set(), true}; + } + if (rel.Properties().empty()) { - auto record = record_factory.NewRecord(); - ProcessPropertiesRel(record, type, "", "", false); + rel_types_properties[rel_type].mandatory = false; // if there is rel with no property, it is not mandatory continue; } + auto &property_info = rel_types_properties.at(rel_type); for (auto &[key, prop] : rel.Properties()) { - auto property_type = mgp::List(); + property_info.properties.emplace(key, std::move(prop)); + if (property_info.mandatory) { + property_info.mandatory = + property_info.properties.size() == 1; // if there is only one property, it is mandatory + } + } + } + + for (auto &[type, property_info] : rel_types_properties) { + std::string type_str = ":`" + std::string(type) + "`"; + for (auto const &prop : property_info.properties) { auto record = record_factory.NewRecord(); - property_type.AppendExtend(mgp::Value(TypeOf(prop.Type()))); - ProcessPropertiesRel(record, type, key, property_type, true); + ProcessPropertiesRel(record, type_str, prop.name, TypeOf(prop.value.Type()), property_info.mandatory); + } + if (property_info.properties.empty()) { + auto record = record_factory.NewRecord(); + ProcessPropertiesRel(record, type_str, "", "", false); } } diff --git a/tests/e2e/query_modules/schema_test.py b/tests/e2e/query_modules/schema_test.py index 515514a74..fbb376a22 100644 --- a/tests/e2e/query_modules/schema_test.py +++ b/tests/e2e/query_modules/schema_test.py @@ -431,7 +431,7 @@ def test_node_type_properties1(): f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", )[0] ) - assert (result) == [":`Activity`", ["Activity"], "location", ["String"], True] + assert (result) == [":`Activity`", ["Activity"], "location", "String", False] result = list( execute_and_fetch_all( @@ -439,7 +439,7 @@ def test_node_type_properties1(): f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", )[1] ) - assert (result) == [":`Activity`", ["Activity"], "name", ["String"], True] + assert (result) == [":`Activity`", ["Activity"], "name", "String", False] result = list( execute_and_fetch_all( @@ -447,7 +447,7 @@ def test_node_type_properties1(): f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", )[2] ) - assert (result) == [":`Dog`", ["Dog"], "name", ["String"], True] + assert (result) == [":`Dog`", ["Dog"], "name", "String", False] result = list( execute_and_fetch_all( @@ -455,7 +455,81 @@ def test_node_type_properties1(): f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", )[3] ) - assert (result) == [":`Dog`", ["Dog"], "owner", ["String"], True] + assert (result) == [":`Dog`", ["Dog"], "owner", "String", False] + + +def test_node_type_properties2(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (d:MyNode) + CREATE (n:MyNode) + """, + ) + result = execute_and_fetch_all( + cursor, + f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", + ) + assert (list(result[0])) == [":`MyNode`", ["MyNode"], "", "", False] + assert (result.__len__()) == 1 + + +def test_node_type_properties3(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (d:Dog {name: 'Rex', owner: 'Carl'}) + CREATE (n:Dog) + """, + ) + result = execute_and_fetch_all( + cursor, + f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", + ) + + assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", False] + assert (list(result[1])) == [":`Dog`", ["Dog"], "owner", "String", False] + assert (result.__len__()) == 2 + + +def test_node_type_properties4(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (n:Label1:Label2 {property1: 'value1', property2: 'value2'}) + CREATE (m:Label2:Label1 {property3: 'value3'}) + """, + ) + result = list( + execute_and_fetch_all( + cursor, + f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", + ) + ) + assert (list(result[0])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property1", "String", False] + assert (list(result[1])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property2", "String", False] + assert (list(result[2])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property3", "String", False] + assert (result.__len__()) == 3 + + +def test_node_type_properties5(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (d:Dog {name: 'Rex'}) + """, + ) + result = execute_and_fetch_all( + cursor, + f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];", + ) + + assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", True] + assert (result.__len__()) == 1 def test_rel_type_properties1(): @@ -473,5 +547,38 @@ def test_rel_type_properties1(): assert (result) == [":`LOVES`", "", "", False] +def test_rel_type_properties2(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (d:Dog {name: 'Rex', owner: 'Carl'})-[l:LOVES]->(a:Activity {name: 'Running', location: 'Zadar'}) + CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'}) + """, + ) + result = execute_and_fetch_all( + cursor, + f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;", + ) + assert (list(result[0])) == [":`LOVES`", "duration", "Int", False] + assert (result.__len__()) == 1 + + +def test_rel_type_properties3(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + """ + CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'}) + """, + ) + result = execute_and_fetch_all( + cursor, + f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;", + ) + assert (list(result[0])) == [":`LOVES`", "duration", "Int", True] + assert (result.__len__()) == 1 + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-rA"])) From 71e76cc980e289e0f33f5e7d8bbec3a3f3784576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Pu=C5=A1i=C4=87?= Date: Mon, 18 Dec 2023 23:31:00 +0100 Subject: [PATCH 50/52] =?UTF-8?q?Fix=20unresolved=20host=20errors=20by=20s?= =?UTF-8?q?witching=20to=20using=20jemalloc=E2=80=99s=20primary=20URL=20(#?= =?UTF-8?q?1579)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/setup.sh b/libs/setup.sh index 028797679..2f968f71c 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -259,7 +259,7 @@ repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" " # jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756 JEMALLOC_COMMIT_VERSION="5.2.1" -repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION" +repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION" # this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR pushd jemalloc From 04fb92dce894e3bcd0004eab01ebdd99267199c5 Mon Sep 17 00:00:00 2001 From: Andi Date: Tue, 19 Dec 2023 11:13:05 +0100 Subject: [PATCH 51/52] Fix memory bug Alloc vs. Free (#1570) --- src/memory/global_memory_control.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory/global_memory_control.cpp b/src/memory/global_memory_control.cpp index 57d97bcaa..ab75435ca 100644 --- a/src/memory/global_memory_control.cpp +++ b/src/memory/global_memory_control.cpp @@ -119,7 +119,7 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz memgraph::utils::total_memory_tracker.Alloc(static_cast(length)); if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] { - GetQueriesMemoryControl().TrackFreeOnCurrentThread(size); + GetQueriesMemoryControl().TrackAllocOnCurrentThread(size); } return false; From 4ef86efb6fb17ae70774d588b5cbc2990c00e242 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:09:43 +0100 Subject: [PATCH 52/52] Fix memgraph crash on telemetry server and no file permissions (#1566) --- src/telemetry/telemetry.cpp | 8 +++++++- src/utils/stat.hpp | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/telemetry/telemetry.cpp b/src/telemetry/telemetry.cpp index 1635dda99..e5e779f31 100644 --- a/src/telemetry/telemetry.cpp +++ b/src/telemetry/telemetry.cpp @@ -110,7 +110,13 @@ void Telemetry::CollectData(const std::string &event) { { std::lock_guard guard(lock_); for (auto &collector : collectors_) { - data[collector.first] = collector.second(); + try { + data[collector.first] = collector.second(); + } catch (std::exception &e) { + spdlog::warn(fmt::format( + "Unknwon exception occured on in telemetry server {}, please contact support on https://memgr.ph/unknown ", + e.what())); + } } } if (event == "") { diff --git a/src/utils/stat.hpp b/src/utils/stat.hpp index 7e4fab29b..4c2eec6d6 100644 --- a/src/utils/stat.hpp +++ b/src/utils/stat.hpp @@ -25,17 +25,32 @@ namespace memgraph::utils { static constexpr int64_t VM_MAX_MAP_COUNT_DEFAULT{-1}; /// Returns the number of bytes a directory is using on disk. If the given path -/// isn't a directory, zero will be returned. +/// isn't a directory, zero will be returned. If there are some files with +/// wrong permission, it will be skipped template inline uint64_t GetDirDiskUsage(const std::filesystem::path &path) { if (!std::filesystem::is_directory(path)) return 0; + if (!utils::HasReadAccess(path)) { + spdlog::warn( + "Skipping directory path on collecting directory disk usage '{}' because it is not readable, check file " + "ownership and read permissions!", + path); + return 0; + } uint64_t size = 0; - for (auto &p : std::filesystem::directory_iterator(path)) { + for (const auto &p : std::filesystem::directory_iterator(path)) { if (IgnoreSymlink && std::filesystem::is_symlink(p)) continue; if (std::filesystem::is_directory(p)) { size += GetDirDiskUsage(p); } else if (std::filesystem::is_regular_file(p)) { + if (!utils::HasReadAccess(p)) { + spdlog::warn( + "Skipping file path on collecting directory disk usage '{}' because it is not readable, check file " + "ownership and read permissions!", + p); + continue; + } size += std::filesystem::file_size(p); } }