From b4d6dc09306cb767ff98916655554a1a9d004102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Wed, 21 Sep 2022 18:25:51 +0200 Subject: [PATCH] Add proper transaction handling (#550) --- .gitignore | 1 + src/coordinator/hybrid_logical_clock.hpp | 8 + src/query/v2/db_accessor.hpp | 6 +- src/query/v2/dump.cpp | 58 - src/query/v2/dump.hpp | 3 - src/query/v2/interpreter.cpp | 465 +------ src/query/v2/plan/operator.cpp | 1 + src/query/v2/requests.hpp | 13 +- src/query/v2/stream/streams.cpp | 4 +- src/storage/v3/CMakeLists.txt | 23 +- src/storage/v3/bindings/db_accessor.hpp | 6 - src/storage/v3/commit_log.cpp | 108 -- src/storage/v3/commit_log.hpp | 78 -- src/storage/v3/config.hpp | 28 +- src/storage/v3/constraints.cpp | 414 ------ src/storage/v3/constraints.hpp | 202 --- src/storage/v3/delta.hpp | 74 +- src/storage/v3/durability/durability.cpp | 346 ----- src/storage/v3/durability/durability.hpp | 112 -- src/storage/v3/durability/exceptions.hpp | 23 - src/storage/v3/durability/marker.hpp | 106 -- src/storage/v3/durability/metadata.hpp | 75 - src/storage/v3/durability/paths.hpp | 50 - src/storage/v3/durability/serialization.cpp | 468 ------- src/storage/v3/durability/serialization.hpp | 143 -- src/storage/v3/durability/snapshot.cpp | 1002 -------------- src/storage/v3/durability/snapshot.hpp | 77 -- src/storage/v3/durability/version.hpp | 37 - src/storage/v3/durability/wal.cpp | 1013 -------------- src/storage/v3/durability/wal.hpp | 262 ---- src/storage/v3/edge_accessor.hpp | 5 +- src/storage/v3/indices.cpp | 52 +- src/storage/v3/indices.hpp | 35 +- src/storage/v3/mvcc.hpp | 33 +- src/storage/v3/replication/.gitignore | 2 - src/storage/v3/replication/config.hpp | 44 - src/storage/v3/replication/enums.hpp | 19 - .../v3/replication/replication_client.cpp | 617 --------- .../v3/replication/replication_client.hpp | 203 --- .../v3/replication/replication_server.cpp | 570 -------- .../v3/replication/replication_server.hpp | 47 - src/storage/v3/replication/rpc.lcp | 74 - src/storage/v3/replication/serialization.cpp | 149 -- src/storage/v3/replication/serialization.hpp | 80 -- src/storage/v3/replication/slk.cpp | 169 --- src/storage/v3/replication/slk.hpp | 41 - src/storage/v3/shard.cpp | 1220 +++-------------- src/storage/v3/shard.hpp | 224 +-- src/storage/v3/shard_rsm.cpp | 74 +- src/storage/v3/shard_rsm.hpp | 3 +- src/storage/v3/transaction.hpp | 48 +- src/storage/v3/vertex_accessor.cpp | 10 +- src/storage/v3/vertex_accessor.hpp | 10 +- src/utils/logging.hpp | 2 +- tests/simulation/shard_rsm.cpp | 41 +- tests/unit/CMakeLists.txt | 3 + tests/unit/storage_v3.cpp | 470 ++++--- tests/unit/storage_v3_edge.cpp | 370 ++--- tests/unit/storage_v3_expr.cpp | 11 +- tests/unit/storage_v3_indices.cpp | 361 +++-- tests/unit/storage_v3_isolation_level.cpp | 142 ++ tests/unit/storage_v3_vertex_accessors.cpp | 37 +- 62 files changed, 1360 insertions(+), 9012 deletions(-) delete mode 100644 src/storage/v3/commit_log.cpp delete mode 100644 src/storage/v3/commit_log.hpp delete mode 100644 src/storage/v3/constraints.cpp delete mode 100644 src/storage/v3/constraints.hpp delete mode 100644 src/storage/v3/durability/durability.cpp delete mode 100644 src/storage/v3/durability/durability.hpp delete mode 100644 src/storage/v3/durability/exceptions.hpp delete mode 100644 src/storage/v3/durability/marker.hpp delete mode 100644 src/storage/v3/durability/metadata.hpp delete mode 100644 src/storage/v3/durability/paths.hpp delete mode 100644 src/storage/v3/durability/serialization.cpp delete mode 100644 src/storage/v3/durability/serialization.hpp delete mode 100644 src/storage/v3/durability/snapshot.cpp delete mode 100644 src/storage/v3/durability/snapshot.hpp delete mode 100644 src/storage/v3/durability/version.hpp delete mode 100644 src/storage/v3/durability/wal.cpp delete mode 100644 src/storage/v3/durability/wal.hpp delete mode 100644 src/storage/v3/replication/.gitignore delete mode 100644 src/storage/v3/replication/config.hpp delete mode 100644 src/storage/v3/replication/enums.hpp delete mode 100644 src/storage/v3/replication/replication_client.cpp delete mode 100644 src/storage/v3/replication/replication_client.hpp delete mode 100644 src/storage/v3/replication/replication_server.cpp delete mode 100644 src/storage/v3/replication/replication_server.hpp delete mode 100644 src/storage/v3/replication/rpc.lcp delete mode 100644 src/storage/v3/replication/serialization.cpp delete mode 100644 src/storage/v3/replication/serialization.hpp delete mode 100644 src/storage/v3/replication/slk.cpp delete mode 100644 src/storage/v3/replication/slk.hpp create mode 100644 tests/unit/storage_v3_isolation_level.cpp diff --git a/.gitignore b/.gitignore index 6ab822041..217d687c5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ src/durability/single_node/state_delta.hpp src/durability/single_node_ha/state_delta.hpp src/query/frontend/semantic/symbol.hpp src/query/v2/frontend/semantic/symbol.hpp +src/expr/semantic/symbol.hpp src/query/distributed/frontend/semantic/symbol_serialization.hpp src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp src/query/distributed/plan/ops.hpp diff --git a/src/coordinator/hybrid_logical_clock.hpp b/src/coordinator/hybrid_logical_clock.hpp index 75d3a2fbb..caea92425 100644 --- a/src/coordinator/hybrid_logical_clock.hpp +++ b/src/coordinator/hybrid_logical_clock.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "io/time.hpp" namespace memgraph::coordinator { @@ -22,7 +24,13 @@ struct Hlc { uint64_t logical_id; Time coordinator_wall_clock; + auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; } + bool operator==(const Hlc &other) const = default; + bool operator<(const Hlc &other) const = default; + bool operator==(const uint64_t other) const { return logical_id == other; } + bool operator<(const uint64_t other) const { return logical_id < other; } + bool operator>=(const uint64_t other) const { return logical_id >= other; } }; } // namespace memgraph::coordinator diff --git a/src/query/v2/db_accessor.hpp b/src/query/v2/db_accessor.hpp index 217ecda0e..a43cdf18f 100644 --- a/src/query/v2/db_accessor.hpp +++ b/src/query/v2/db_accessor.hpp @@ -263,8 +263,6 @@ class DbAccessor final { return std::nullopt; } - void FinalizeTransaction() { accessor_->FinalizeTransaction(); } - VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); } VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) { @@ -376,7 +374,7 @@ class DbAccessor final { void AdvanceCommand() { accessor_->AdvanceCommand(); } - utils::BasicResult Commit() { return accessor_->Commit(); } + void Commit() { return accessor_->Commit(coordinator::Hlc{}); } void Abort() { accessor_->Abort(); } @@ -407,8 +405,6 @@ class DbAccessor final { storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); } - storage::v3::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); } - const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); } storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); } diff --git a/src/query/v2/dump.cpp b/src/query/v2/dump.cpp index 0f1682cb5..bf20abd0d 100644 --- a/src/query/v2/dump.cpp +++ b/src/query/v2/dump.cpp @@ -269,10 +269,6 @@ PullPlanDump::PullPlanDump(DbAccessor *dba) CreateLabelIndicesPullChunk(), // Dump all label property indices CreateLabelPropertyIndicesPullChunk(), - // Dump all existence constraints - CreateExistenceConstraintsPullChunk(), - // Dump all unique constraints - CreateUniqueConstraintsPullChunk(), // Create internal index for faster edge creation CreateInternalIndexPullChunk(), // Dump all vertices @@ -364,60 +360,6 @@ PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() { }; } -PullPlanDump::PullChunk PullPlanDump::CreateExistenceConstraintsPullChunk() { - return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the construction of constraint vectors - if (!constraints_info_) { - constraints_info_.emplace(dba_->ListAllConstraints()); - } - - const auto &existence = constraints_info_->existence; - size_t local_counter = 0; - while (global_index < existence.size() && (!n || local_counter < *n)) { - const auto &constraint = existence[global_index]; - std::ostringstream os; - DumpExistenceConstraint(&os, dba_, constraint.first, constraint.second); - stream->Result({TypedValue(os.str())}); - - ++global_index; - ++local_counter; - } - - if (global_index == existence.size()) { - return local_counter; - } - - return std::nullopt; - }; -} - -PullPlanDump::PullChunk PullPlanDump::CreateUniqueConstraintsPullChunk() { - return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { - // Delay the construction of constraint vectors - if (!constraints_info_) { - constraints_info_.emplace(dba_->ListAllConstraints()); - } - - const auto &unique = constraints_info_->unique; - size_t local_counter = 0; - while (global_index < unique.size() && (!n || local_counter < *n)) { - const auto &constraint = unique[global_index]; - std::ostringstream os; - DumpUniqueConstraint(&os, dba_, constraint.first, constraint.second); - stream->Result({TypedValue(os.str())}); - - ++global_index; - ++local_counter; - } - - if (global_index == unique.size()) { - return local_counter; - } - - return std::nullopt; - }; -} - PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() { return [this](AnyStream *stream, std::optional) mutable -> std::optional { if (vertices_iterable_.begin() != vertices_iterable_.end()) { diff --git a/src/query/v2/dump.hpp b/src/query/v2/dump.hpp index de8018724..3c8130d8d 100644 --- a/src/query/v2/dump.hpp +++ b/src/query/v2/dump.hpp @@ -32,7 +32,6 @@ struct PullPlanDump { query::v2::DbAccessor *dba_ = nullptr; std::optional indices_info_ = std::nullopt; - std::optional constraints_info_ = std::nullopt; using VertexAccessorIterable = decltype(std::declval().Vertices(storage::v3::View::OLD)); using VertexAccessorIterableIterator = decltype(std::declval().begin()); @@ -55,8 +54,6 @@ struct PullPlanDump { PullChunk CreateLabelIndicesPullChunk(); PullChunk CreateLabelPropertyIndicesPullChunk(); - PullChunk CreateExistenceConstraintsPullChunk(); - PullChunk CreateUniqueConstraintsPullChunk(); PullChunk CreateInternalIndexPullChunk(); PullChunk CreateVertexPullChunk(); PullChunk CreateEdgePullChunk(); diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp index f93471179..7347c71a4 100644 --- a/src/query/v2/interpreter.cpp +++ b/src/query/v2/interpreter.cpp @@ -129,135 +129,25 @@ std::optional GetOptionalStringValue(query::v2::Expression *express class ReplQueryHandler final : public query::v2::ReplicationQueryHandler { public: - explicit ReplQueryHandler(storage::v3::Shard *db) : db_(db) {} + explicit ReplQueryHandler(storage::v3::Shard * /*db*/) {} /// @throw QueryRuntimeException if an error ocurred. - void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { - if (replication_role == ReplicationQuery::ReplicationRole::MAIN) { - if (!db_->SetMainReplicationRole()) { - throw QueryRuntimeException("Couldn't set role to main!"); - } - } - if (replication_role == ReplicationQuery::ReplicationRole::REPLICA) { - if (!port || *port < 0 || *port > std::numeric_limits::max()) { - throw QueryRuntimeException("Port number invalid!"); - } - if (!db_->SetReplicaRole( - io::network::Endpoint(query::v2::kDefaultReplicationServerIp, static_cast(*port)))) { - throw QueryRuntimeException("Couldn't set role to replica!"); - } - } - } + void SetReplicationRole(ReplicationQuery::ReplicationRole /*replication_role*/, + std::optional /*port*/) override {} /// @throw QueryRuntimeException if an error ocurred. - ReplicationQuery::ReplicationRole ShowReplicationRole() const override { - switch (db_->GetReplicationRole()) { - case storage::v3::ReplicationRole::MAIN: - return ReplicationQuery::ReplicationRole::MAIN; - case storage::v3::ReplicationRole::REPLICA: - return ReplicationQuery::ReplicationRole::REPLICA; - } - throw QueryRuntimeException("Couldn't show replication role - invalid role set!"); - } + ReplicationQuery::ReplicationRole ShowReplicationRole() const override { return {}; } /// @throw QueryRuntimeException if an error ocurred. - void RegisterReplica(const std::string &name, const std::string &socket_address, - const ReplicationQuery::SyncMode sync_mode, const std::optional timeout, - const std::chrono::seconds replica_check_frequency) override { - if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { - // replica can't register another replica - throw QueryRuntimeException("Replica can't register another replica!"); - } - - storage::v3::replication::ReplicationMode repl_mode; - switch (sync_mode) { - case ReplicationQuery::SyncMode::ASYNC: { - repl_mode = storage::v3::replication::ReplicationMode::ASYNC; - break; - } - case ReplicationQuery::SyncMode::SYNC: { - repl_mode = storage::v3::replication::ReplicationMode::SYNC; - break; - } - } - - auto maybe_ip_and_port = - io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::v2::kDefaultReplicationPort); - if (maybe_ip_and_port) { - auto [ip, port] = *maybe_ip_and_port; - auto ret = db_->RegisterReplica( - name, {std::move(ip), port}, repl_mode, - {.timeout = timeout, .replica_check_frequency = replica_check_frequency, .ssl = std::nullopt}); - if (ret.HasError()) { - throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name)); - } - } else { - throw QueryRuntimeException("Invalid socket address!"); - } - } + void RegisterReplica(const std::string & /*name*/, const std::string & /*socket_address*/, + const ReplicationQuery::SyncMode /*sync_mode*/, const std::optional /*timeout*/, + const std::chrono::seconds /*replica_check_frequency*/) override {} /// @throw QueryRuntimeException if an error ocurred. - void DropReplica(const std::string &replica_name) override { - if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { - // replica can't unregister a replica - throw QueryRuntimeException("Replica can't unregister a replica!"); - } - if (!db_->UnregisterReplica(replica_name)) { - throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name)); - } - } + void DropReplica(const std::string & /*replica_name*/) override {} using Replica = ReplicationQueryHandler::Replica; - std::vector ShowReplicas() const override { - if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { - // replica can't show registered replicas (it shouldn't have any) - throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!"); - } - - auto repl_infos = db_->ReplicasInfo(); - std::vector replicas; - replicas.reserve(repl_infos.size()); - - const auto from_info = [](const auto &repl_info) -> Replica { - Replica replica; - replica.name = repl_info.name; - replica.socket_address = repl_info.endpoint.SocketAddress(); - switch (repl_info.mode) { - case storage::v3::replication::ReplicationMode::SYNC: - replica.sync_mode = ReplicationQuery::SyncMode::SYNC; - break; - case storage::v3::replication::ReplicationMode::ASYNC: - replica.sync_mode = ReplicationQuery::SyncMode::ASYNC; - break; - } - if (repl_info.timeout) { - replica.timeout = *repl_info.timeout; - } - - switch (repl_info.state) { - case storage::v3::replication::ReplicaState::READY: - replica.state = ReplicationQuery::ReplicaState::READY; - break; - case storage::v3::replication::ReplicaState::REPLICATING: - replica.state = ReplicationQuery::ReplicaState::REPLICATING; - break; - case storage::v3::replication::ReplicaState::RECOVERY: - replica.state = ReplicationQuery::ReplicaState::RECOVERY; - break; - case storage::v3::replication::ReplicaState::INVALID: - replica.state = ReplicationQuery::ReplicaState::INVALID; - break; - } - - return replica; - }; - - std::transform(repl_infos.begin(), repl_infos.end(), std::back_inserter(replicas), from_info); - return replicas; - } - - private: - storage::v3::Shard *db_; + std::vector ShowReplicas() const override { return {}; } }; /// returns false if the replication role can't be set /// @throw QueryRuntimeException if an error ocurred. @@ -1158,8 +1048,8 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) in_explicit_transaction_ = true; expect_rollback_ = false; - db_accessor_ = - std::make_unique(interpreter_context_->db->Access(GetIsolationLevelOverride())); + db_accessor_ = std::make_unique( + interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride())); execution_db_accessor_.emplace(db_accessor_.get()); if (interpreter_context_->trigger_store.HasTriggers()) { @@ -1581,48 +1471,22 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, InterpreterContext *interpreter_context, DbAccessor *dba) { - if (in_explicit_transaction) { - throw LockPathModificationInMulticommandTxException(); - } - - auto *lock_path_query = utils::Downcast(parsed_query.query); - - return PreparedQuery{{}, - std::move(parsed_query.required_privileges), - [interpreter_context, action = lock_path_query->action_]( - AnyStream *stream, std::optional n) -> std::optional { - switch (action) { - case LockPathQuery::Action::LOCK_PATH: - if (!interpreter_context->db->LockPath()) { - throw QueryRuntimeException("Failed to lock the data directory"); - } - break; - case LockPathQuery::Action::UNLOCK_PATH: - if (!interpreter_context->db->UnlockPath()) { - throw QueryRuntimeException("Failed to unlock the data directory"); - } - break; - } - return QueryHandlerResult::COMMIT; - }, - RWType::NONE}; + throw SemanticException("LockPath query is not supported!"); } PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, - InterpreterContext *interpreter_context) { + InterpreterContext * /*interpreter_context*/) { if (in_explicit_transaction) { throw FreeMemoryModificationInMulticommandTxException(); } - return PreparedQuery{ - {}, - std::move(parsed_query.required_privileges), - [interpreter_context](AnyStream *stream, std::optional n) -> std::optional { - interpreter_context->db->FreeMemory(); - memory::PurgeUnusedMemory(); - return QueryHandlerResult::COMMIT; - }, - RWType::NONE}; + return PreparedQuery{{}, + std::move(parsed_query.required_privileges), + [](AnyStream * /*stream*/, std::optional /*n*/) -> std::optional { + memory::PurgeUnusedMemory(); + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; } TriggerEventType ToTriggerEventType(const TriggerQuery::EventType event_type) { @@ -1835,24 +1699,7 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_explicit_transaction, InterpreterContext *interpreter_context) { - if (in_explicit_transaction) { - throw CreateSnapshotInMulticommandTxException(); - } - - return PreparedQuery{ - {}, - std::move(parsed_query.required_privileges), - [interpreter_context](AnyStream *stream, std::optional n) -> std::optional { - if (auto maybe_error = interpreter_context->db->CreateSnapshot(); maybe_error.HasError()) { - switch (maybe_error.GetError()) { - case storage::v3::Shard::CreateSnapshotError::DisabledForReplica: - throw utils::BasicException( - "Failed to create a snapshot. Replica instances are not allowed to create them."); - } - } - return QueryHandlerResult::COMMIT; - }, - RWType::NONE}; + throw SemanticException("CreateSnapshot query is not supported!"); } PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, DbAccessor *dba) { @@ -1920,7 +1767,6 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa {TypedValue("edge_count"), TypedValue(static_cast(info.edge_count))}, {TypedValue("average_degree"), TypedValue(info.average_degree)}, {TypedValue("memory_usage"), TypedValue(static_cast(info.memory_usage))}, - {TypedValue("disk_usage"), TypedValue(static_cast(info.disk_usage))}, {TypedValue("memory_allocated"), TypedValue(static_cast(utils::total_memory_tracker.Amount()))}, {TypedValue("allocation_limit"), TypedValue(static_cast(utils::total_memory_tracker.HardLimit()))}}; @@ -1945,28 +1791,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa }; break; case InfoQuery::InfoType::CONSTRAINT: - header = {"constraint type", "label", "properties"}; - handler = [interpreter_context] { - auto *db = interpreter_context->db; - auto info = db->ListAllConstraints(); - std::vector> results; - results.reserve(info.existence.size() + info.unique.size()); - for (const auto &item : info.existence) { - results.push_back({TypedValue("exists"), TypedValue(db->LabelToName(item.first)), - TypedValue(db->PropertyToName(item.second))}); - } - for (const auto &item : info.unique) { - std::vector properties; - properties.reserve(item.second.size()); - for (const auto &property : item.second) { - properties.emplace_back(db->PropertyToName(property)); - } - results.push_back( - {TypedValue("unique"), TypedValue(db->LabelToName(item.first)), TypedValue(std::move(properties))}); - } - return std::pair{results, QueryHandlerResult::NOTHING}; - }; - break; + throw SemanticException("Constraints are not yet supported!"); } return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), @@ -1990,185 +1815,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction, std::vector *notifications, InterpreterContext *interpreter_context) { - if (in_explicit_transaction) { - throw ConstraintInMulticommandTxException(); - } - - auto *constraint_query = utils::Downcast(parsed_query.query); - std::function handler; - - const auto label = interpreter_context->NameToLabelId(constraint_query->constraint_.label.name); - std::vector properties; - std::vector properties_string; - properties.reserve(constraint_query->constraint_.properties.size()); - properties_string.reserve(constraint_query->constraint_.properties.size()); - for (const auto &prop : constraint_query->constraint_.properties) { - properties.push_back(interpreter_context->NameToPropertyId(prop.name)); - properties_string.push_back(prop.name); - } - auto properties_stringified = utils::Join(properties_string, ", "); - - Notification constraint_notification(SeverityLevel::INFO); - switch (constraint_query->action_type_) { - case ConstraintQuery::ActionType::CREATE: { - constraint_notification.code = NotificationCode::CREATE_CONSTRAINT; - - switch (constraint_query->constraint_.type) { - case Constraint::Type::NODE_KEY: - throw utils::NotYetImplemented("Node key constraints"); - case Constraint::Type::EXISTS: - if (properties.empty() || properties.size() > 1) { - throw SyntaxException("Exactly one property must be used for existence constraints."); - } - constraint_notification.title = fmt::format("Created EXISTS constraint on label {} on properties {}.", - constraint_query->constraint_.label.name, properties_stringified); - handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, - properties_stringified = std::move(properties_stringified), - properties = std::move(properties)](Notification &constraint_notification) { - auto res = interpreter_context->db->CreateExistenceConstraint(label, properties[0]); - if (res.HasError()) { - auto violation = res.GetError(); - auto label_name = interpreter_context->db->LabelToName(violation.label); - MG_ASSERT(violation.properties.size() == 1U); - auto property_name = interpreter_context->db->PropertyToName(*violation.properties.begin()); - throw QueryRuntimeException( - "Unable to create existence constraint :{}({}), because an " - "existing node violates it.", - label_name, property_name); - } - if (res.HasValue() && !res.GetValue()) { - constraint_notification.code = NotificationCode::EXISTANT_CONSTRAINT; - constraint_notification.title = fmt::format( - "Constraint EXISTS on label {} on properties {} already exists.", label_name, properties_stringified); - } - }; - break; - case Constraint::Type::UNIQUE: - std::set property_set; - for (const auto &property : properties) { - property_set.insert(property); - } - if (property_set.size() != properties.size()) { - throw SyntaxException("The given set of properties contains duplicates."); - } - constraint_notification.title = - fmt::format("Created UNIQUE constraint on label {} on properties {}.", - constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, - properties_stringified = std::move(properties_stringified), - property_set = std::move(property_set)](Notification &constraint_notification) { - auto res = interpreter_context->db->CreateUniqueConstraint(label, property_set); - if (res.HasError()) { - auto violation = res.GetError(); - auto label_name = interpreter_context->db->LabelToName(violation.label); - std::stringstream property_names_stream; - utils::PrintIterable(property_names_stream, violation.properties, ", ", - [&interpreter_context](auto &stream, const auto &prop) { - stream << interpreter_context->db->PropertyToName(prop); - }); - throw QueryRuntimeException( - "Unable to create unique constraint :{}({}), because an " - "existing node violates it.", - label_name, property_names_stream.str()); - } - switch (res.GetValue()) { - case storage::v3::UniqueConstraints::CreationStatus::EMPTY_PROPERTIES: - throw SyntaxException( - "At least one property must be used for unique " - "constraints."); - case storage::v3::UniqueConstraints::CreationStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED: - throw SyntaxException( - "Too many properties specified. Limit of {} properties " - "for unique constraints is exceeded.", - storage::v3::kUniqueConstraintsMaxProperties); - case storage::v3::UniqueConstraints::CreationStatus::ALREADY_EXISTS: - constraint_notification.code = NotificationCode::EXISTANT_CONSTRAINT; - constraint_notification.title = - fmt::format("Constraint UNIQUE on label {} on properties {} already exists.", label_name, - properties_stringified); - break; - case storage::v3::UniqueConstraints::CreationStatus::SUCCESS: - break; - } - }; - break; - } - } break; - case ConstraintQuery::ActionType::DROP: { - constraint_notification.code = NotificationCode::DROP_CONSTRAINT; - - switch (constraint_query->constraint_.type) { - case Constraint::Type::NODE_KEY: - throw utils::NotYetImplemented("Node key constraints"); - case Constraint::Type::EXISTS: - if (properties.empty() || properties.size() > 1) { - throw SyntaxException("Exactly one property must be used for existence constraints."); - } - constraint_notification.title = - fmt::format("Dropped EXISTS constraint on label {} on properties {}.", - constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, - properties_stringified = std::move(properties_stringified), - properties = std::move(properties)](Notification &constraint_notification) { - if (!interpreter_context->db->DropExistenceConstraint(label, properties[0])) { - constraint_notification.code = NotificationCode::NONEXISTANT_CONSTRAINT; - constraint_notification.title = fmt::format( - "Constraint EXISTS on label {} on properties {} doesn't exist.", label_name, properties_stringified); - } - return std::vector>(); - }; - break; - case Constraint::Type::UNIQUE: - std::set property_set; - for (const auto &property : properties) { - property_set.insert(property); - } - if (property_set.size() != properties.size()) { - throw SyntaxException("The given set of properties contains duplicates."); - } - constraint_notification.title = - fmt::format("Dropped UNIQUE constraint on label {} on properties {}.", - constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, - properties_stringified = std::move(properties_stringified), - property_set = std::move(property_set)](Notification &constraint_notification) { - auto res = interpreter_context->db->DropUniqueConstraint(label, property_set); - switch (res) { - case storage::v3::UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES: - throw SyntaxException( - "At least one property must be used for unique " - "constraints."); - break; - case storage::v3::UniqueConstraints::DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED: - throw SyntaxException( - "Too many properties specified. Limit of {} properties for " - "unique constraints is exceeded.", - storage::v3::kUniqueConstraintsMaxProperties); - break; - case storage::v3::UniqueConstraints::DeletionStatus::NOT_FOUND: - constraint_notification.code = NotificationCode::NONEXISTANT_CONSTRAINT; - constraint_notification.title = - fmt::format("Constraint UNIQUE on label {} on properties {} doesn't exist.", label_name, - properties_stringified); - break; - case storage::v3::UniqueConstraints::DeletionStatus::SUCCESS: - break; - } - return std::vector>(); - }; - } - } break; - } - - return PreparedQuery{{}, - std::move(parsed_query.required_privileges), - [handler = std::move(handler), constraint_notification = std::move(constraint_notification), - notifications](AnyStream * /*stream*/, std::optional /*n*/) mutable { - handler(constraint_notification); - notifications->push_back(constraint_notification); - return QueryHandlerResult::COMMIT; - }, - RWType::NONE}; + throw SemanticException("Constraint query is not supported!"); } PreparedQuery PrepareSchemaQuery(ParsedQuery parsed_query, bool in_explicit_transaction, @@ -2262,8 +1909,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, (utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query))) { - db_accessor_ = - std::make_unique(interpreter_context_->db->Access(GetIsolationLevelOverride())); + db_accessor_ = std::make_unique( + interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride())); execution_db_accessor_.emplace(db_accessor_.get()); if (utils::Downcast(parsed_query.query) && interpreter_context_->trigger_store.HasTriggers()) { @@ -2345,13 +1992,6 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, UpdateTypeCount(rw_type); - if (const auto query_type = query_execution->prepared_query->rw_type; - interpreter_context_->db->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA && - (query_type == RWType::W || query_type == RWType::RW)) { - query_execution = nullptr; - throw QueryException("Write query forbidden on the replica!"); - } - return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid}; } catch (const utils::BasicException &) { EventCounter::IncrementCounter(EventCounter::FailedQuery); @@ -2378,7 +2018,7 @@ void RunTriggersIndividually(const utils::SkipList &triggers, Interpret utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize}; // create a new transaction for each trigger - auto storage_acc = interpreter_context->db->Access(); + auto storage_acc = interpreter_context->db->Access(coordinator::Hlc{}); DbAccessor db_accessor{&storage_acc}; trigger_context.AdaptForAccessor(&db_accessor); @@ -2391,29 +2031,7 @@ void RunTriggersIndividually(const utils::SkipList &triggers, Interpret continue; } - auto maybe_constraint_violation = db_accessor.Commit(); - if (maybe_constraint_violation.HasError()) { - const auto &constraint_violation = maybe_constraint_violation.GetError(); - switch (constraint_violation.type) { - case storage::v3::ConstraintViolation::Type::EXISTENCE: { - const auto &label_name = db_accessor.LabelToName(constraint_violation.label); - MG_ASSERT(constraint_violation.properties.size() == 1U); - const auto &property_name = db_accessor.PropertyToName(*constraint_violation.properties.begin()); - spdlog::warn("Trigger '{}' failed to commit due to existence constraint violation on :{}({})", trigger.Name(), - label_name, property_name); - break; - } - case storage::v3::ConstraintViolation::Type::UNIQUE: { - const auto &label_name = db_accessor.LabelToName(constraint_violation.label); - std::stringstream property_names_stream; - utils::PrintIterable(property_names_stream, constraint_violation.properties, ", ", - [&](auto &stream, const auto &prop) { stream << db_accessor.PropertyToName(prop); }); - spdlog::warn("Trigger '{}' failed to commit due to unique constraint violation on :{}({})", trigger.Name(), - label_name, property_names_stream.str()); - break; - } - } - } + db_accessor.Commit(); } } } // namespace @@ -2454,33 +2072,7 @@ void Interpreter::Commit() { trigger_context_collector_.reset(); }; - auto maybe_constraint_violation = db_accessor_->Commit(); - if (maybe_constraint_violation.HasError()) { - const auto &constraint_violation = maybe_constraint_violation.GetError(); - switch (constraint_violation.type) { - case storage::v3::ConstraintViolation::Type::EXISTENCE: { - auto label_name = execution_db_accessor_->LabelToName(constraint_violation.label); - MG_ASSERT(constraint_violation.properties.size() == 1U); - auto property_name = execution_db_accessor_->PropertyToName(*constraint_violation.properties.begin()); - reset_necessary_members(); - throw QueryException("Unable to commit due to existence constraint violation on :{}({})", label_name, - property_name); - break; - } - case storage::v3::ConstraintViolation::Type::UNIQUE: { - auto label_name = execution_db_accessor_->LabelToName(constraint_violation.label); - std::stringstream property_names_stream; - utils::PrintIterable( - property_names_stream, constraint_violation.properties, ", ", - [this](auto &stream, const auto &prop) { stream << execution_db_accessor_->PropertyToName(prop); }); - reset_necessary_members(); - throw QueryException("Unable to commit due to unique constraint violation on :{}({})", label_name, - property_names_stream.str()); - break; - } - } - } - + db_accessor_->Commit(coordinator::Hlc{}); // The ordered execution of after commit triggers is heavily depending on the exclusiveness of db_accessor_->Commit(): // only one of the transactions can be commiting at the same time, so when the commit is finished, that transaction // probably will schedule its after commit triggers, because the other transactions that want to commit are still @@ -2492,7 +2084,6 @@ void Interpreter::Commit() { user_transaction = std::shared_ptr(std::move(db_accessor_))]() mutable { RunTriggersIndividually(interpreter_context->trigger_store.AfterCommitTriggers(), interpreter_context, std::move(trigger_context)); - user_transaction->FinalizeTransaction(); SPDLOG_DEBUG("Finished executing after commit triggers"); // NOLINT(bugprone-lambda-function-name) }); } diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp index 1c15ed4b9..0c6ac1d56 100644 --- a/src/query/v2/plan/operator.cpp +++ b/src/query/v2/plan/operator.cpp @@ -47,6 +47,7 @@ #include "utils/likely.hpp" #include "utils/logging.hpp" #include "utils/memory.hpp" +#include "utils/message.hpp" #include "utils/pmr/unordered_map.hpp" #include "utils/pmr/unordered_set.hpp" #include "utils/pmr/vector.hpp" diff --git a/src/query/v2/requests.hpp b/src/query/v2/requests.hpp index 07770f5aa..05dc6f7a9 100644 --- a/src/query/v2/requests.hpp +++ b/src/query/v2/requests.hpp @@ -496,12 +496,21 @@ struct UpdateEdgesResponse { bool success; }; +struct CommitRequest { + Hlc transaction_id; + Hlc commit_timestamp; +}; + +struct CommitResponse { + bool success; +}; + using ReadRequests = std::variant; using ReadResponses = std::variant; using WriteRequests = std::variant; + CreateEdgesRequest, DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest>; using WriteResponses = std::variant; + CreateEdgesResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>; } // namespace memgraph::msgs diff --git a/src/query/v2/stream/streams.cpp b/src/query/v2/stream/streams.cpp index 5fdd28cb5..b7e6abe61 100644 --- a/src/query/v2/stream/streams.cpp +++ b/src/query/v2/stream/streams.cpp @@ -490,7 +490,7 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std total_retries = interpreter_context_->config.stream_transaction_conflict_retries, retry_interval = interpreter_context_->config.stream_transaction_retry_interval]( const std::vector &messages) mutable { - auto accessor = interpreter_context->db->Access(); + auto accessor = interpreter_context->db->Access(coordinator::Hlc{}); EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size()); CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name); @@ -738,7 +738,7 @@ TransformationResult Streams::Check(const std::string &stream_name, std::optiona auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name, &transformation_name = transformation_name, &result, &test_result](const std::vector &messages) mutable { - auto accessor = interpreter_context->db->Access(); + auto accessor = interpreter_context->db->Access(coordinator::Hlc{}); CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name); auto result_row = std::vector(); diff --git a/src/storage/v3/CMakeLists.txt b/src/storage/v3/CMakeLists.txt index c7dc091b6..73f64a49d 100644 --- a/src/storage/v3/CMakeLists.txt +++ b/src/storage/v3/CMakeLists.txt @@ -6,13 +6,7 @@ add_custom_target(generate_lcp_ast_storage_v3 DEPENDS ${generated_lcp_storage_v3 set(storage_v3_src_files ${lcp_storage_v3_cpp_files} - commit_log.cpp - constraints.cpp temporal.cpp - durability/durability.cpp - durability/serialization.cpp - durability/snapshot.cpp - durability/wal.cpp edge_accessor.cpp indices.cpp key_store.cpp @@ -26,21 +20,6 @@ set(storage_v3_src_files shard_rsm.cpp storage.cpp) -# #### Replication ##### -define_add_lcp(add_lcp_storage lcp_storage_cpp_files generated_lcp_storage_files) - -add_lcp_storage(replication/rpc.lcp SLK_SERIALIZE) - -add_custom_target(generate_lcp_storage_v3 DEPENDS ${generated_lcp_storage_files}) - -set(storage_v3_src_files - ${storage_v3_src_files} - replication/replication_client.cpp - replication/replication_server.cpp - replication/serialization.cpp - replication/slk.cpp - ${lcp_storage_cpp_files}) - # ###################### find_package(gflags REQUIRED) find_package(Threads REQUIRED) @@ -51,4 +30,4 @@ target_link_libraries(mg-storage-v3 Threads::Threads mg-utils gflags) target_include_directories(mg-storage-v3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings) add_dependencies(mg-storage-v3 generate_lcp_storage) -target_link_libraries(mg-storage-v3 mg-rpc mg-slk mg-expr) +target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io) diff --git a/src/storage/v3/bindings/db_accessor.hpp b/src/storage/v3/bindings/db_accessor.hpp index b382719da..28d603e3d 100644 --- a/src/storage/v3/bindings/db_accessor.hpp +++ b/src/storage/v3/bindings/db_accessor.hpp @@ -60,8 +60,6 @@ class DbAccessor final { return std::nullopt; } - void FinalizeTransaction() { accessor_->FinalizeTransaction(); } - VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); } VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) { @@ -169,8 +167,6 @@ class DbAccessor final { void AdvanceCommand() { accessor_->AdvanceCommand(); } - utils::BasicResult Commit() { return accessor_->Commit(); } - void Abort() { accessor_->Abort(); } bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); } @@ -200,8 +196,6 @@ class DbAccessor final { storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); } - storage::v3::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); } - const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); } storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); } diff --git a/src/storage/v3/commit_log.cpp b/src/storage/v3/commit_log.cpp deleted file mode 100644 index 4ce4586be..000000000 --- a/src/storage/v3/commit_log.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/commit_log.hpp" -#include "utils/memory.hpp" - -namespace memgraph::storage::v3 { -CommitLog::CommitLog() : allocator_(utils::NewDeleteResource()) {} - -CommitLog::CommitLog(uint64_t oldest_active) - : head_start_{oldest_active / kIdsInBlock * kIdsInBlock}, - next_start_{head_start_ + kIdsInBlock}, - allocator_{utils::NewDeleteResource()} { - head_ = allocator_.allocate(1); - allocator_.construct(head_); - - // set all the previous ids - const auto field_idx = (oldest_active % kIdsInBlock) / kIdsInField; - for (size_t i = 0; i < field_idx; ++i) { - head_->field[i] = std::numeric_limits::max(); - } - - const auto idx_in_field = oldest_active % kIdsInField; - if (idx_in_field != 0) { - head_->field[field_idx] = std::numeric_limits::max(); - head_->field[field_idx] >>= kIdsInField - idx_in_field; - } - - oldest_active_ = oldest_active; -} - -CommitLog::~CommitLog() { - while (head_) { - Block *tmp = head_->next; - head_->~Block(); - allocator_.deallocate(head_, 1); - head_ = tmp; - } -} - -void CommitLog::MarkFinished(uint64_t id) { - Block *block = FindOrCreateBlock(id); - block->field[(id % kIdsInBlock) / kIdsInField] |= 1ULL << (id % kIdsInField); - if (id == oldest_active_) { - UpdateOldestActive(); - } -} - -uint64_t CommitLog::OldestActive() const noexcept { return oldest_active_; } - -void CommitLog::UpdateOldestActive() { - while (head_) { - // This is necessary for amortized constant complexity. If we always start - // from the 0th field, the amount of steps we make through each block is - // quadratic in kBlockSize. - uint64_t start_field = oldest_active_ >= head_start_ ? (oldest_active_ - head_start_) / kIdsInField : 0; - for (uint64_t i = start_field; i < kBlockSize; ++i) { - if (head_->field[i] != std::numeric_limits::max()) { - // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) - oldest_active_ = head_start_ + i * kIdsInField + __builtin_ffsl(static_cast(~head_->field[i])) - 1; - return; - } - } - - // All IDs in this block are marked, we can delete it now. - Block *tmp = head_->next; - head_->~Block(); - allocator_.deallocate(head_, 1); - head_ = tmp; - head_start_ += kIdsInBlock; - } - - oldest_active_ = next_start_; -} - -CommitLog::Block *CommitLog::FindOrCreateBlock(const uint64_t id) { - if (!head_) { - head_ = allocator_.allocate(1); - allocator_.construct(head_); - head_start_ = next_start_; - next_start_ += kIdsInBlock; - } - - Block *current = head_; - uint64_t current_start = head_start_; - - while (id >= current_start + kIdsInBlock) { - if (!current->next) { - current->next = allocator_.allocate(1); - allocator_.construct(current->next); - next_start_ += kIdsInBlock; - } - - current = current->next; - current_start += kIdsInBlock; - } - - return current; -} -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/commit_log.hpp b/src/storage/v3/commit_log.hpp deleted file mode 100644 index 0ef3817dd..000000000 --- a/src/storage/v3/commit_log.hpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -/// @file commit_log.hpp -#pragma once - -#include -#include - -#include "utils/memory.hpp" -#include "utils/spin_lock.hpp" - -namespace memgraph::storage::v3 { - -/// This class keeps track of finalized transactions to provide info on the -/// oldest active transaction (minimal transaction ID which could still be -/// active). -/// -/// Basically, it is a set which, at the beginning, contains all transaction -/// IDs and supports two operations: remove an ID from the set (\ref -/// SetFinished) and retrieve the minimal ID still in the set (\ref -/// OldestActive). -/// -/// This class is thread-safe. -class CommitLog final { - public: - // TODO(mtomic): use pool allocator for blocks - CommitLog(); - /// Create a commit log which has the oldest active id set to - /// oldest_active - /// @param oldest_active the oldest active id - explicit CommitLog(uint64_t oldest_active); - - CommitLog(const CommitLog &) = delete; - CommitLog &operator=(const CommitLog &) = delete; - CommitLog(CommitLog &&) = delete; - CommitLog &operator=(CommitLog &&) = delete; - - ~CommitLog(); - - /// Mark a transaction as finished. - /// @throw std::bad_alloc - void MarkFinished(uint64_t id); - - /// Retrieve the oldest transaction still not marked as finished. - uint64_t OldestActive() const noexcept; - - private: - static constexpr uint64_t kBlockSize = 8192; - static constexpr uint64_t kIdsInField = sizeof(uint64_t) * 8; - static constexpr uint64_t kIdsInBlock = kBlockSize * kIdsInField; - - struct Block { - Block *next{nullptr}; - uint64_t field[kBlockSize]{}; - }; - - void UpdateOldestActive(); - - /// @throw std::bad_alloc - Block *FindOrCreateBlock(uint64_t id); - - Block *head_{nullptr}; - uint64_t head_start_{0}; - uint64_t next_start_{0}; - uint64_t oldest_active_{0}; - utils::Allocator allocator_; -}; - -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/config.hpp b/src/storage/v3/config.hpp index bb99905df..05179c6b0 100644 --- a/src/storage/v3/config.hpp +++ b/src/storage/v3/config.hpp @@ -14,6 +14,8 @@ #include #include #include + +#include "io/time.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/isolation_level.hpp" #include "storage/v3/transaction.hpp" @@ -24,36 +26,14 @@ namespace memgraph::storage::v3 { /// the storage. This class also defines the default behavior. struct Config { struct Gc { - // TODO(antaljanosbenjamin): How to handle garbage collection? - enum class Type { NONE }; - - Type type{Type::NONE}; - std::chrono::milliseconds interval{std::chrono::milliseconds(1000)}; + // Interval after which the committed deltas are cleaned up + io::Duration reclamation_interval{}; } gc; struct Items { bool properties_on_edges{true}; } items; - struct Durability { - enum class SnapshotWalMode { DISABLED, PERIODIC_SNAPSHOT, PERIODIC_SNAPSHOT_WITH_WAL }; - - std::filesystem::path storage_directory{"storage"}; - - bool recover_on_startup{false}; - - SnapshotWalMode snapshot_wal_mode{SnapshotWalMode::DISABLED}; - - std::chrono::milliseconds snapshot_interval{std::chrono::minutes(2)}; - uint64_t snapshot_retention_count{3}; - - uint64_t wal_file_size_kibibytes{static_cast(20 * 1024)}; - uint64_t wal_file_flush_every_n_tx{100000}; - - bool snapshot_on_exit{false}; - - } durability; - struct Transaction { IsolationLevel isolation_level{IsolationLevel::SNAPSHOT_ISOLATION}; } transaction; diff --git a/src/storage/v3/constraints.cpp b/src/storage/v3/constraints.cpp deleted file mode 100644 index 17315165a..000000000 --- a/src/storage/v3/constraints.cpp +++ /dev/null @@ -1,414 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/constraints.hpp" - -#include -#include -#include - -#include "storage/v3/mvcc.hpp" -#include "storage/v3/vertex.hpp" -#include "utils/logging.hpp" - -namespace memgraph::storage::v3 { -namespace { - -/// Helper function that determines position of the given `property` in the -/// sorted `property_array` using binary search. In the case that `property` -/// cannot be found, `std::nullopt` is returned. -std::optional FindPropertyPosition(const PropertyIdArray &property_array, PropertyId property) { - const auto *it = std::lower_bound(property_array.values, property_array.values + property_array.size, property); - if (it == property_array.values + property_array.size || *it != property) { - return std::nullopt; - } - - return it - property_array.values; -} - -/// Helper function for validating unique constraints on commit. Returns true if -/// the last committed version of the given vertex contains the given label and -/// set of property values. This function should be called when commit lock is -/// active. -bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std::set &properties, - const std::vector &value_array, const Transaction &transaction, - uint64_t commit_timestamp) { - MG_ASSERT(properties.size() == value_array.size(), "Invalid database state!"); - - PropertyIdArray property_array(properties.size()); - bool current_value_equal_to_value[kUniqueConstraintsMaxProperties]; - memset(current_value_equal_to_value, 0, sizeof(current_value_equal_to_value)); - - // Since the commit lock is active, any transaction that tries to write to - // a vertex which is part of the given `transaction` will result in a - // serialization error. But, note that the given `vertex`'s data does not have - // to be modified in the current `transaction`, meaning that a guard lock to - // access vertex's data is still necessary because another active transaction - // could modify it in the meantime. - Delta *delta{nullptr}; - bool deleted{false}; - bool has_label{false}; - { - delta = vertex.delta; - deleted = vertex.deleted; - has_label = VertexHasLabel(vertex, label); - - size_t i = 0; - for (const auto &property : properties) { - current_value_equal_to_value[i] = vertex.properties.IsPropertyEqual(property, value_array[i]); - property_array.values[i] = property; - i++; - } - } - - while (delta != nullptr) { - auto ts = delta->timestamp->load(std::memory_order_acquire); - if (ts < commit_timestamp || ts == transaction.transaction_id) { - break; - } - - switch (delta->action) { - case Delta::Action::SET_PROPERTY: { - auto pos = FindPropertyPosition(property_array, delta->property.key); - if (pos) { - current_value_equal_to_value[*pos] = delta->property.value == value_array[*pos]; - } - break; - } - case Delta::Action::DELETE_OBJECT: { - MG_ASSERT(!deleted, "Invalid database state!"); - deleted = true; - break; - } - case Delta::Action::RECREATE_OBJECT: { - MG_ASSERT(deleted, "Invalid database state!"); - deleted = false; - break; - } - case Delta::Action::ADD_LABEL: { - if (delta->label == label) { - MG_ASSERT(!has_label, "Invalid database state!"); - has_label = true; - break; - } - } - case Delta::Action::REMOVE_LABEL: { - if (delta->label == label) { - MG_ASSERT(has_label, "Invalid database state!"); - has_label = false; - break; - } - } - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - break; - } - - delta = delta->next.load(std::memory_order_acquire); - } - - for (size_t i = 0; i < properties.size(); ++i) { - if (!current_value_equal_to_value[i]) { - return false; - } - } - - return !deleted && has_label; -} - -/// Helper function for unique constraint garbage collection. Returns true if -/// there's a reachable version of the vertex that has the given label and -/// property values. -bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std::set &properties, - const std::vector &values, uint64_t timestamp) { - MG_ASSERT(properties.size() == values.size(), "Invalid database state!"); - - PropertyIdArray property_array(properties.size()); - bool current_value_equal_to_value[kUniqueConstraintsMaxProperties]; - memset(current_value_equal_to_value, 0, sizeof(current_value_equal_to_value)); - - bool has_label{false}; - bool deleted{false}; - Delta *delta{nullptr}; - { - has_label = VertexHasLabel(vertex, label); - deleted = vertex.deleted; - delta = vertex.delta; - - size_t i = 0; - for (const auto &property : properties) { - current_value_equal_to_value[i] = vertex.properties.IsPropertyEqual(property, values[i]); - property_array.values[i] = property; - i++; - } - } - - { - bool all_values_match = true; - for (size_t i = 0; i < values.size(); ++i) { - if (!current_value_equal_to_value[i]) { - all_values_match = false; - break; - } - } - if (!deleted && has_label && all_values_match) { - return true; - } - } - - while (delta != nullptr) { - auto ts = delta->timestamp->load(std::memory_order_acquire); - if (ts < timestamp) { - break; - } - switch (delta->action) { - case Delta::Action::ADD_LABEL: - if (delta->label == label) { - MG_ASSERT(!has_label, "Invalid database state!"); - has_label = true; - } - break; - case Delta::Action::REMOVE_LABEL: - if (delta->label == label) { - MG_ASSERT(has_label, "Invalid database state!"); - has_label = false; - } - break; - case Delta::Action::SET_PROPERTY: { - auto pos = FindPropertyPosition(property_array, delta->property.key); - if (pos) { - current_value_equal_to_value[*pos] = delta->property.value == values[*pos]; - } - break; - } - case Delta::Action::RECREATE_OBJECT: { - MG_ASSERT(deleted, "Invalid database state!"); - deleted = false; - break; - } - case Delta::Action::DELETE_OBJECT: { - MG_ASSERT(!deleted, "Invalid database state!"); - deleted = true; - break; - } - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - break; - } - - bool all_values_match = true; - for (size_t i = 0; i < values.size(); ++i) { - if (!current_value_equal_to_value[i]) { - all_values_match = false; - break; - } - } - if (!deleted && has_label && all_values_match) { - return true; - } - delta = delta->next.load(std::memory_order_acquire); - } - return false; -} - -/// Helper function that, given the set of `properties`, extracts corresponding -/// property values from the `vertex`. -/// @throw std::bad_alloc -std::optional> ExtractPropertyValues(const Vertex &vertex, - const std::set &properties) { - std::vector value_array; - value_array.reserve(properties.size()); - for (const auto &prop : properties) { - auto value = vertex.properties.GetProperty(prop); - if (value.IsNull()) { - return std::nullopt; - } - value_array.emplace_back(std::move(value)); - } - return std::move(value_array); -} - -} // namespace - -bool operator==(const ConstraintViolation &lhs, const ConstraintViolation &rhs) { - return lhs.type == rhs.type && lhs.label == rhs.label && lhs.properties == rhs.properties; -} - -bool UniqueConstraints::Entry::operator<(const Entry &rhs) const { - if (values < rhs.values) { - return true; - } - if (rhs.values < values) { - return false; - } - return std::make_tuple(vertex, timestamp) < std::make_tuple(rhs.vertex, rhs.timestamp); -} - -bool UniqueConstraints::Entry::operator==(const Entry &rhs) const { - return values == rhs.values && vertex == rhs.vertex && timestamp == rhs.timestamp; -} - -bool UniqueConstraints::Entry::operator<(const std::vector &rhs) const { return values < rhs; } - -bool UniqueConstraints::Entry::operator==(const std::vector &rhs) const { return values == rhs; } - -void UniqueConstraints::UpdateBeforeCommit(const Vertex *vertex, const Transaction &tx) { - for (auto &[label_props, storage] : constraints_) { - if (!VertexHasLabel(*vertex, label_props.first)) { - continue; - } - auto values = ExtractPropertyValues(*vertex, label_props.second); - if (values) { - auto acc = storage.access(); - acc.insert(Entry{std::move(*values), vertex, tx.start_timestamp}); - } - } -} - -utils::BasicResult UniqueConstraints::CreateConstraint( - LabelId label, const std::set &properties, VerticesSkipList::Accessor vertices) { - if (properties.empty()) { - return CreationStatus::EMPTY_PROPERTIES; - } - if (properties.size() > kUniqueConstraintsMaxProperties) { - return CreationStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED; - } - - auto [constraint, emplaced] = - constraints_.emplace(std::piecewise_construct, std::forward_as_tuple(label, properties), std::forward_as_tuple()); - - if (!emplaced) { - // Constraint already exists. - return CreationStatus::ALREADY_EXISTS; - } - - bool violation_found = false; - - { - auto acc = constraint->second.access(); - - for (const auto &lo_vertex : vertices) { - const auto &vertex = lo_vertex.vertex; - if (vertex.deleted || !VertexHasLabel(vertex, label)) { - continue; - } - auto values = ExtractPropertyValues(vertex, properties); - if (!values) { - continue; - } - - // Check whether there already is a vertex with the same values for the - // given label and property. - auto it = acc.find_equal_or_greater(*values); - if (it != acc.end() && it->values == *values) { - violation_found = true; - break; - } - - acc.insert(Entry{std::move(*values), &vertex, 0}); - } - } - - if (violation_found) { - // In the case of the violation, storage for the current constraint has to - // be removed. - constraints_.erase(constraint); - return ConstraintViolation{ConstraintViolation::Type::UNIQUE, label, properties}; - } - return CreationStatus::SUCCESS; -} - -UniqueConstraints::DeletionStatus UniqueConstraints::DropConstraint(LabelId label, - const std::set &properties) { - if (properties.empty()) { - return UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES; - } - if (properties.size() > kUniqueConstraintsMaxProperties) { - return UniqueConstraints::DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED; - } - if (constraints_.erase({label, properties}) > 0) { - return UniqueConstraints::DeletionStatus::SUCCESS; - } - return UniqueConstraints::DeletionStatus::NOT_FOUND; -} - -std::optional UniqueConstraints::Validate(const Vertex &vertex, const Transaction &tx, - uint64_t commit_timestamp) const { - if (vertex.deleted) { - return std::nullopt; - } - for (const auto &[label_props, storage] : constraints_) { - const auto &label = label_props.first; - const auto &properties = label_props.second; - if (!VertexHasLabel(vertex, label)) { - continue; - } - - auto value_array = ExtractPropertyValues(vertex, properties); - if (!value_array) { - continue; - } - auto acc = storage.access(); - auto it = acc.find_equal_or_greater(*value_array); - for (; it != acc.end(); ++it) { - if (*value_array < it->values) { - break; - } - - // The `vertex` that is going to be committed violates a unique constraint - // if it's different than a vertex indexed in the list of constraints and - // has the same label and property value as the last committed version of - // the vertex from the list. - if (&vertex != it->vertex && - LastCommittedVersionHasLabelProperty(*it->vertex, label, properties, *value_array, tx, commit_timestamp)) { - return ConstraintViolation{ConstraintViolation::Type::UNIQUE, label, properties}; - } - } - } - return std::nullopt; -} - -std::vector>> UniqueConstraints::ListConstraints() const { - std::vector>> ret; - ret.reserve(constraints_.size()); - for (const auto &[label_props, _] : constraints_) { - ret.push_back(label_props); - } - return ret; -} - -void UniqueConstraints::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) { - for (auto &[label_props, storage] : constraints_) { - auto acc = storage.access(); - for (auto it = acc.begin(); it != acc.end();) { - auto next_it = it; - ++next_it; - - if (it->timestamp >= oldest_active_start_timestamp) { - it = next_it; - continue; - } - - if ((next_it != acc.end() && it->vertex == next_it->vertex && it->values == next_it->values) || - !AnyVersionHasLabelProperty(*it->vertex, label_props.first, label_props.second, it->values, - oldest_active_start_timestamp)) { - acc.remove(*it); - } - it = next_it; - } - } -} - -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/constraints.hpp b/src/storage/v3/constraints.hpp deleted file mode 100644 index 899a21b6e..000000000 --- a/src/storage/v3/constraints.hpp +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -#include "storage/v3/id_types.hpp" -#include "storage/v3/transaction.hpp" -#include "storage/v3/vertex.hpp" -#include "storage/v3/vertices_skip_list.hpp" -#include "utils/logging.hpp" -#include "utils/result.hpp" -#include "utils/skip_list.hpp" - -namespace memgraph::storage::v3 { - -// NOLINTNEXTLINE(misc-definitions-in-headers) -const size_t kUniqueConstraintsMaxProperties = 32; - -/// Utility class to store data in a fixed size array. The array is used -/// instead of `std::vector` to avoid `std::bad_alloc` exception where not -/// necessary. -template -struct FixedCapacityArray { - size_t size; - T values[kUniqueConstraintsMaxProperties]; - - explicit FixedCapacityArray(size_t array_size) : size(array_size) { - MG_ASSERT(size <= kUniqueConstraintsMaxProperties, "Invalid array size!"); - } -}; - -using PropertyIdArray = FixedCapacityArray; - -struct ConstraintViolation { - enum class Type { - EXISTENCE, - UNIQUE, - }; - - Type type; - LabelId label; - - // While multiple properties are supported by unique constraints, the - // `properties` set will always have exactly one element in the case of - // existence constraint violation. - std::set properties; -}; - -bool operator==(const ConstraintViolation &lhs, const ConstraintViolation &rhs); - -class UniqueConstraints { - private: - struct Entry { - std::vector values; - const Vertex *vertex; - uint64_t timestamp; - - bool operator<(const Entry &rhs) const; - bool operator==(const Entry &rhs) const; - - bool operator<(const std::vector &rhs) const; - bool operator==(const std::vector &rhs) const; - }; - - public: - /// Status for creation of unique constraints. - /// Note that this does not cover the case when the constraint is violated. - enum class CreationStatus { - SUCCESS, - ALREADY_EXISTS, - EMPTY_PROPERTIES, - PROPERTIES_SIZE_LIMIT_EXCEEDED, - }; - - /// Status for deletion of unique constraints. - enum class DeletionStatus { - SUCCESS, - NOT_FOUND, - EMPTY_PROPERTIES, - PROPERTIES_SIZE_LIMIT_EXCEEDED, - }; - - /// Indexes the given vertex for relevant labels and properties. - /// This method should be called before committing and validating vertices - /// against unique constraints. - /// @throw std::bad_alloc - void UpdateBeforeCommit(const Vertex *vertex, const Transaction &tx); - - /// Creates unique constraint on the given `label` and a list of `properties`. - /// Returns constraint violation if there are multiple vertices with the same - /// label and property values. Returns `CreationStatus::ALREADY_EXISTS` if - /// constraint already existed, `CreationStatus::EMPTY_PROPERTIES` if the - /// given list of properties is empty, - /// `CreationStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED` if the list of properties - /// exceeds the maximum allowed number of properties, and - /// `CreationStatus::SUCCESS` on success. - /// @throw std::bad_alloc - utils::BasicResult CreateConstraint(LabelId label, - const std::set &properties, - VerticesSkipList::Accessor vertices); - - /// Deletes the specified constraint. Returns `DeletionStatus::NOT_FOUND` if - /// there is not such constraint in the storage, - /// `DeletionStatus::EMPTY_PROPERTIES` if the given set of `properties` is - /// empty, `DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED` if the given set - /// of `properties` exceeds the maximum allowed number of properties, and - /// `DeletionStatus::SUCCESS` on success. - DeletionStatus DropConstraint(LabelId label, const std::set &properties); - - bool ConstraintExists(LabelId label, const std::set &properties) { - return constraints_.find({label, properties}) != constraints_.end(); - } - - /// Validates the given vertex against unique constraints before committing. - /// This method should be called while commit lock is active with - /// `commit_timestamp` being a potential commit timestamp of the transaction. - /// @throw std::bad_alloc - std::optional Validate(const Vertex &vertex, const Transaction &tx, - uint64_t commit_timestamp) const; - - std::vector>> ListConstraints() const; - - /// GC method that removes outdated entries from constraints' storages. - void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp); - - void Clear() { constraints_.clear(); } - - private: - std::map>, utils::SkipList> constraints_; -}; - -struct Constraints { - std::vector> existence_constraints; - UniqueConstraints unique_constraints; -}; - -/// Adds a unique constraint to `constraints`. Returns true if the constraint -/// was successfully added, false if it already exists and a -/// `ConstraintViolation` if there is an existing vertex violating the -/// constraint. -/// -/// @throw std::bad_alloc -/// @throw std::length_error -inline utils::BasicResult CreateExistenceConstraint(Constraints *constraints, LabelId label, - PropertyId property, - VerticesSkipList::Accessor vertices) { - if (utils::Contains(constraints->existence_constraints, std::make_pair(label, property))) { - return false; - } - for (const auto &lgo_vertex : vertices) { - const auto &vertex = lgo_vertex.vertex; - if (!vertex.deleted && VertexHasLabel(vertex, label) && !vertex.properties.HasProperty(property)) { - return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set{property}}; - } - } - constraints->existence_constraints.emplace_back(label, property); - return true; -} - -/// Removes a unique constraint from `constraints`. Returns true if the -/// constraint was removed, and false if it doesn't exist. -inline bool DropExistenceConstraint(Constraints *constraints, LabelId label, PropertyId property) { - auto it = std::find(constraints->existence_constraints.begin(), constraints->existence_constraints.end(), - std::make_pair(label, property)); - if (it == constraints->existence_constraints.end()) { - return false; - } - constraints->existence_constraints.erase(it); - return true; -} - -/// Verifies that the given vertex satisfies all existence constraints. Returns -/// `std::nullopt` if all checks pass, and `ConstraintViolation` describing the -/// violated constraint otherwise. -[[nodiscard]] inline std::optional ValidateExistenceConstraints(const Vertex &vertex, - const Constraints &constraints) { - for (const auto &[label, property] : constraints.existence_constraints) { - if (!vertex.deleted && VertexHasLabel(vertex, label) && !vertex.properties.HasProperty(property)) { - return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set{property}}; - } - } - return std::nullopt; -} - -/// Returns a list of all created existence constraints. -inline std::vector> ListExistenceConstraints(const Constraints &constraints) { - return constraints.existence_constraints; -} - -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/delta.hpp b/src/storage/v3/delta.hpp index 93238d6aa..9548be146 100644 --- a/src/storage/v3/delta.hpp +++ b/src/storage/v3/delta.hpp @@ -11,9 +11,7 @@ #pragma once -#include #include - #include "storage/v3/edge_ref.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/property_value.hpp" @@ -26,6 +24,7 @@ namespace memgraph::storage::v3 { struct Vertex; struct Edge; struct Delta; +struct CommitInfo; // This class stores one of three pointers (`Delta`, `Vertex` and `Edge`) // without using additional memory for storing the type. The type is stored in @@ -63,31 +62,30 @@ class PreviousPtr { Edge *edge{nullptr}; }; - PreviousPtr() : storage_(0) {} + PreviousPtr() {} - PreviousPtr(const PreviousPtr &other) noexcept : storage_(other.storage_.load(std::memory_order_acquire)) {} + PreviousPtr(const PreviousPtr &other) noexcept : storage_(other.storage_) {} PreviousPtr(PreviousPtr &&) = delete; PreviousPtr &operator=(const PreviousPtr &) = delete; PreviousPtr &operator=(PreviousPtr &&) = delete; ~PreviousPtr() = default; Pointer Get() const { - uintptr_t value = storage_.load(std::memory_order_acquire); - if (value == 0) { + if (storage_ == 0) { return {}; } - uintptr_t type = value & kMask; + uintptr_t type = storage_ & kMask; if (type == kDelta) { // NOLINTNEXTLINE(performance-no-int-to-ptr) - return Pointer{reinterpret_cast(value & ~kMask)}; + return Pointer{reinterpret_cast(storage_ & ~kMask)}; } if (type == kVertex) { // NOLINTNEXTLINE(performance-no-int-to-ptr) - return Pointer{reinterpret_cast(value & ~kMask)}; + return Pointer{reinterpret_cast(storage_ & ~kMask)}; } if (type == kEdge) { // NOLINTNEXTLINE(performance-no-int-to-ptr) - return Pointer{reinterpret_cast(value & ~kMask)}; + return Pointer{reinterpret_cast(storage_ & ~kMask)}; } LOG_FATAL("Invalid pointer type!"); } @@ -95,23 +93,23 @@ class PreviousPtr { void Set(Delta *delta) { auto value = reinterpret_cast(delta); MG_ASSERT((value & kMask) == 0, "Invalid pointer!"); - storage_.store(value | kDelta, std::memory_order_release); + storage_ = value | kDelta; } void Set(Vertex *vertex) { auto value = reinterpret_cast(vertex); MG_ASSERT((value & kMask) == 0, "Invalid pointer!"); - storage_.store(value | kVertex, std::memory_order_release); + storage_ = value | kVertex; } void Set(Edge *edge) { auto value = reinterpret_cast(edge); MG_ASSERT((value & kMask) == 0, "Invalid pointer!"); - storage_.store(value | kEdge, std::memory_order_release); + storage_ = value | kEdge; } private: - std::atomic storage_; + uintptr_t storage_{0}; }; inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { @@ -159,47 +157,47 @@ struct Delta { struct RemoveInEdgeTag {}; struct RemoveOutEdgeTag {}; - Delta(DeleteObjectTag /*unused*/, std::atomic *timestamp, uint64_t command_id) - : action(Action::DELETE_OBJECT), timestamp(timestamp), command_id(command_id) {} + Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id) + : action(Action::DELETE_OBJECT), commit_info(commit_info), command_id(command_id) {} - Delta(RecreateObjectTag /*unused*/, std::atomic *timestamp, uint64_t command_id) - : action(Action::RECREATE_OBJECT), timestamp(timestamp), command_id(command_id) {} + Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id) + : action(Action::RECREATE_OBJECT), commit_info(commit_info), command_id(command_id) {} - Delta(AddLabelTag /*unused*/, LabelId label, std::atomic *timestamp, uint64_t command_id) - : action(Action::ADD_LABEL), timestamp(timestamp), command_id(command_id), label(label) {} + Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id) + : action(Action::ADD_LABEL), commit_info(commit_info), command_id(command_id), label(label) {} - Delta(RemoveLabelTag /*unused*/, LabelId label, std::atomic *timestamp, uint64_t command_id) - : action(Action::REMOVE_LABEL), timestamp(timestamp), command_id(command_id), label(label) {} + Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id) + : action(Action::REMOVE_LABEL), commit_info(commit_info), command_id(command_id), label(label) {} - Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, std::atomic *timestamp, + Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, CommitInfo *commit_info, uint64_t command_id) - : action(Action::SET_PROPERTY), timestamp(timestamp), command_id(command_id), property({key, value}) {} + : action(Action::SET_PROPERTY), commit_info(commit_info), command_id(command_id), property({key, value}) {} - Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, - std::atomic *timestamp, uint64_t command_id) + Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info, + uint64_t command_id) : action(Action::ADD_IN_EDGE), - timestamp(timestamp), + commit_info(commit_info), command_id(command_id), vertex_edge({edge_type, std::move(vertex_id), edge}) {} - Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, - std::atomic *timestamp, uint64_t command_id) + Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info, + uint64_t command_id) : action(Action::ADD_OUT_EDGE), - timestamp(timestamp), + commit_info(commit_info), command_id(command_id), vertex_edge({edge_type, std::move(vertex_id), edge}) {} - Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, - std::atomic *timestamp, uint64_t command_id) + Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info, + uint64_t command_id) : action(Action::REMOVE_IN_EDGE), - timestamp(timestamp), + commit_info(commit_info), command_id(command_id), vertex_edge({edge_type, std::move(vertex_id), edge}) {} - Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, - std::atomic *timestamp, uint64_t command_id) + Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info, + uint64_t command_id) : action(Action::REMOVE_OUT_EDGE), - timestamp(timestamp), + commit_info(commit_info), command_id(command_id), vertex_edge({edge_type, std::move(vertex_id), edge}) {} @@ -230,10 +228,10 @@ struct Delta { Action action; // TODO: optimize with in-place copy - std::atomic *timestamp; + CommitInfo *commit_info; uint64_t command_id; PreviousPtr prev; - std::atomic next{nullptr}; + Delta *next{nullptr}; union { LabelId label; diff --git a/src/storage/v3/durability/durability.cpp b/src/storage/v3/durability/durability.cpp deleted file mode 100644 index 65ef294f0..000000000 --- a/src/storage/v3/durability/durability.cpp +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/durability/durability.hpp" - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include "storage/v3/durability/paths.hpp" -#include "storage/v3/durability/snapshot.hpp" -#include "storage/v3/durability/wal.hpp" -#include "utils/logging.hpp" -#include "utils/memory_tracker.hpp" -#include "utils/message.hpp" - -namespace memgraph::storage::v3::durability { - -void VerifyStorageDirectoryOwnerAndProcessUserOrDie(const std::filesystem::path &storage_directory) { - // Get the process user ID. - auto process_euid = geteuid(); - - // Get the data directory owner ID. - struct stat statbuf; - auto ret = stat(storage_directory.c_str(), &statbuf); - if (ret != 0 && errno == ENOENT) { - // The directory doesn't currently exist. - return; - } - MG_ASSERT(ret == 0, "Couldn't get stat for '{}' because of: {} ({})", storage_directory, strerror(errno), errno); - auto directory_owner = statbuf.st_uid; - - auto get_username = [](auto uid) { - auto info = getpwuid(uid); - if (!info) return std::to_string(uid); - return std::string(info->pw_name); - }; - - auto user_process = get_username(process_euid); - auto user_directory = get_username(directory_owner); - MG_ASSERT(process_euid == directory_owner, - "The process is running as user {}, but the data directory is " - "owned by user {}. Please start the process as user {}!", - user_process, user_directory, user_directory); -} - -std::vector GetSnapshotFiles(const std::filesystem::path &snapshot_directory, - const std::string_view uuid) { - std::vector snapshot_files; - std::error_code error_code; - if (utils::DirExists(snapshot_directory)) { - for (const auto &item : std::filesystem::directory_iterator(snapshot_directory, error_code)) { - if (!item.is_regular_file()) continue; - try { - auto info = ReadSnapshotInfo(item.path()); - if (uuid.empty() || info.uuid == uuid) { - snapshot_files.emplace_back(item.path(), std::move(info.uuid), info.start_timestamp); - } - } catch (const RecoveryFailure &) { - continue; - } - } - MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); - } - - return snapshot_files; -} - -std::optional> GetWalFiles(const std::filesystem::path &wal_directory, - const std::string_view uuid, - const std::optional current_seq_num) { - if (!utils::DirExists(wal_directory)) return std::nullopt; - - std::vector wal_files; - std::error_code error_code; - for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { - if (!item.is_regular_file()) continue; - try { - auto info = ReadWalInfo(item.path()); - if ((uuid.empty() || info.uuid == uuid) && (!current_seq_num || info.seq_num < *current_seq_num)) - wal_files.emplace_back(info.seq_num, info.from_timestamp, info.to_timestamp, std::move(info.uuid), - std::move(info.epoch_id), item.path()); - } catch (const RecoveryFailure &e) { - spdlog::warn("Failed to read {}", item.path()); - continue; - } - } - MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); - - std::sort(wal_files.begin(), wal_files.end()); - return std::move(wal_files); -} - -// Function used to recover all discovered indices and constraints. The -// indices and constraints must be recovered after the data recovery is done -// to ensure that the indices and constraints are consistent at the end of the -// recovery process. -void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, - Constraints *constraints, VerticesSkipList *vertices) { - spdlog::info("Recreating indices from metadata."); - // Recover label indices. - spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size()); - for (const auto &item : indices_constraints.indices.label) { - if (!indices->label_index.CreateIndex(item, vertices->access())) - throw RecoveryFailure("The label index must be created here!"); - spdlog::info("A label index is recreated from metadata."); - } - spdlog::info("Label indices are recreated."); - - // Recover label+property indices. - spdlog::info("Recreating {} label+property indices from metadata.", - indices_constraints.indices.label_property.size()); - for (const auto &item : indices_constraints.indices.label_property) { - if (!indices->label_property_index.CreateIndex(item.first, item.second, vertices->access())) - throw RecoveryFailure("The label+property index must be created here!"); - spdlog::info("A label+property index is recreated from metadata."); - } - spdlog::info("Label+property indices are recreated."); - spdlog::info("Indices are recreated."); - - spdlog::info("Recreating constraints from metadata."); - // Recover existence constraints. - spdlog::info("Recreating {} existence constraints from metadata.", indices_constraints.constraints.existence.size()); - for (const auto &item : indices_constraints.constraints.existence) { - auto ret = CreateExistenceConstraint(constraints, item.first, item.second, vertices->access()); - if (ret.HasError() || !ret.GetValue()) throw RecoveryFailure("The existence constraint must be created here!"); - spdlog::info("A existence constraint is recreated from metadata."); - } - spdlog::info("Existence constraints are recreated from metadata."); - - // Recover unique constraints. - spdlog::info("Recreating {} unique constraints from metadata.", indices_constraints.constraints.unique.size()); - for (const auto &item : indices_constraints.constraints.unique) { - auto ret = constraints->unique_constraints.CreateConstraint(item.first, item.second, vertices->access()); - if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) - throw RecoveryFailure("The unique constraint must be created here!"); - spdlog::info("A unique constraint is recreated from metadata."); - } - spdlog::info("Unique constraints are recreated from metadata."); - spdlog::info("Constraints are recreated from metadata."); -} - -std::optional RecoverData(const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, std::string *uuid, - std::string *epoch_id, - std::deque> *epoch_history, - VerticesSkipList *vertices, utils::SkipList *edges, uint64_t *edge_count, - NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, - Config::Items items, uint64_t *wal_seq_num) { - utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory, - wal_directory); - if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) { - spdlog::warn(utils::MessageWithLink("Snapshot or WAL directory don't exist, there is nothing to recover.", - "https://memgr.ph/durability")); - return std::nullopt; - } - - auto snapshot_files = GetSnapshotFiles(snapshot_directory); - - RecoveryInfo recovery_info; - RecoveredIndicesAndConstraints indices_constraints; - std::optional snapshot_timestamp; - if (!snapshot_files.empty()) { - spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory); - // Order the files by name - std::sort(snapshot_files.begin(), snapshot_files.end()); - - // UUID used for durability is the UUID of the last snapshot file. - *uuid = snapshot_files.back().uuid; - std::optional recovered_snapshot; - for (auto it = snapshot_files.rbegin(); it != snapshot_files.rend(); ++it) { - const auto &[path, file_uuid, _] = *it; - if (file_uuid != *uuid) { - spdlog::warn("The snapshot file {} isn't related to the latest snapshot file!", path); - continue; - } - spdlog::info("Starting snapshot recovery from {}.", path); - try { - recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, items); - spdlog::info("Snapshot recovery successful!"); - break; - } catch (const RecoveryFailure &e) { - spdlog::warn("Couldn't recover snapshot from {} because of: {}.", path, e.what()); - continue; - } - } - MG_ASSERT(recovered_snapshot, - "The database is configured to recover on startup, but couldn't " - "recover using any of the specified snapshots! Please inspect them " - "and restart the database."); - recovery_info = recovered_snapshot->recovery_info; - indices_constraints = std::move(recovered_snapshot->indices_constraints); - snapshot_timestamp = recovered_snapshot->snapshot_info.start_timestamp; - *epoch_id = std::move(recovered_snapshot->snapshot_info.epoch_id); - - if (!utils::DirExists(wal_directory)) { - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices); - return recovered_snapshot->recovery_info; - } - } else { - spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory); - std::error_code error_code; - if (!utils::DirExists(wal_directory)) return std::nullopt; - // We use this smaller struct that contains only a subset of information - // necessary for the rest of the recovery function. - // Also, the struct is sorted primarily on the path it contains. - struct WalFileInfo { - explicit WalFileInfo(std::filesystem::path path, std::string uuid, std::string epoch_id) - : path(std::move(path)), uuid(std::move(uuid)), epoch_id(std::move(epoch_id)) {} - std::filesystem::path path; - std::string uuid; - std::string epoch_id; - // NOLINTNEXTLINE(modernize-use-nullptr): bug in clang-tidy - auto operator<=>(const WalFileInfo &) const = default; - }; - std::vector wal_files; - for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { - if (!item.is_regular_file()) continue; - try { - auto info = ReadWalInfo(item.path()); - wal_files.emplace_back(item.path(), std::move(info.uuid), std::move(info.epoch_id)); - } catch (const RecoveryFailure &e) { - continue; - } - } - MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); - if (wal_files.empty()) { - spdlog::warn(utils::MessageWithLink("No snapshot or WAL file found.", "https://memgr.ph/durability")); - return std::nullopt; - } - std::sort(wal_files.begin(), wal_files.end()); - // UUID used for durability is the UUID of the last WAL file. - // Same for the epoch id. - *uuid = std::move(wal_files.back().uuid); - *epoch_id = std::move(wal_files.back().epoch_id); - } - - auto maybe_wal_files = GetWalFiles(wal_directory, *uuid); - if (!maybe_wal_files) { - spdlog::warn( - utils::MessageWithLink("Couldn't get WAL file info from the WAL directory.", "https://memgr.ph/durability")); - return std::nullopt; - } - - // Array of all discovered WAL files, ordered by sequence number. - auto &wal_files = *maybe_wal_files; - - // By this point we should have recovered from a snapshot, or we should have - // found some WAL files to recover from in the above `else`. This is just a - // sanity check to circumvent the following case: The database didn't recover - // from a snapshot, the above `else` triggered to find the recovery UUID from - // a WAL file. The above `else` has an early exit in case there are no WAL - // files. Because we reached this point there must have been some WAL files - // and we must have some WAL files after this second WAL directory iteration. - MG_ASSERT(snapshot_timestamp || !wal_files.empty(), - "The database didn't recover from a snapshot and didn't find any WAL " - "files that match the last WAL file!"); - - if (!wal_files.empty()) { - spdlog::info("Checking WAL files."); - { - const auto &first_wal = wal_files[0]; - if (first_wal.seq_num != 0) { - // We don't have all WAL files. We need to see whether we need them all. - if (!snapshot_timestamp) { - // We didn't recover from a snapshot and we must have all WAL files - // starting from the first one (seq_num == 0) to be able to recover - // data from them. - LOG_FATAL( - "There are missing prefix WAL files and data can't be " - "recovered without them!"); - } else if (first_wal.from_timestamp >= *snapshot_timestamp) { - // We recovered from a snapshot and we must have at least one WAL file - // that has at least one delta that was created before the snapshot in order to - // verify that nothing is missing from the beginning of the WAL chain. - LOG_FATAL( - "You must have at least one WAL file that contains at least one " - "delta that was created before the snapshot file!"); - } - } - } - std::optional previous_seq_num; - auto last_loaded_timestamp = snapshot_timestamp; - spdlog::info("Trying to load WAL files."); - for (auto &wal_file : wal_files) { - if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) { - LOG_FATAL("You are missing a WAL file with the sequence number {}!", *previous_seq_num + 1); - } - previous_seq_num = wal_file.seq_num; - - if (wal_file.epoch_id != *epoch_id) { - // This way we skip WALs finalized only because of role change. - // We can also set the last timestamp to 0 if last loaded timestamp - // is nullopt as this can only happen if the WAL file with seq = 0 - // does not contain any deltas and we didn't find any snapshots. - if (last_loaded_timestamp) { - epoch_history->emplace_back(wal_file.epoch_id, *last_loaded_timestamp); - } - *epoch_id = std::move(wal_file.epoch_id); - } - try { - auto info = LoadWal(wal_file.path, &indices_constraints, last_loaded_timestamp, vertices, edges, name_id_mapper, - edge_count, items); - recovery_info.next_vertex_id = std::max(recovery_info.next_vertex_id, info.next_vertex_id); - recovery_info.next_edge_id = std::max(recovery_info.next_edge_id, info.next_edge_id); - recovery_info.next_timestamp = std::max(recovery_info.next_timestamp, info.next_timestamp); - - recovery_info.last_commit_timestamp = info.last_commit_timestamp; - } catch (const RecoveryFailure &e) { - LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", wal_file.path, e.what()); - } - - if (recovery_info.next_timestamp != 0) { - last_loaded_timestamp.emplace(recovery_info.next_timestamp - 1); - } - } - // The sequence number needs to be recovered even though `LoadWal` didn't - // load any deltas from that file. - *wal_seq_num = *previous_seq_num + 1; - - spdlog::info("All necessary WAL files are loaded successfully."); - } - - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices); - return recovery_info; -} - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/durability.hpp b/src/storage/v3/durability/durability.hpp deleted file mode 100644 index b84c59b23..000000000 --- a/src/storage/v3/durability/durability.hpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include -#include - -#include "storage/v3/config.hpp" -#include "storage/v3/constraints.hpp" -#include "storage/v3/durability/metadata.hpp" -#include "storage/v3/durability/wal.hpp" -#include "storage/v3/edge.hpp" -#include "storage/v3/indices.hpp" -#include "storage/v3/name_id_mapper.hpp" -#include "storage/v3/vertex.hpp" -#include "utils/skip_list.hpp" - -namespace memgraph::storage::v3::durability { - -/// Verifies that the owner of the storage directory is the same user that -/// started the current process. If the verification fails, the process is -/// killed (`CHECK` failure). -void VerifyStorageDirectoryOwnerAndProcessUserOrDie(const std::filesystem::path &storage_directory); - -// Used to capture the snapshot's data related to durability -struct SnapshotDurabilityInfo { - explicit SnapshotDurabilityInfo(std::filesystem::path path, std::string uuid, const uint64_t start_timestamp) - : path(std::move(path)), uuid(std::move(uuid)), start_timestamp(start_timestamp) {} - - std::filesystem::path path; - std::string uuid; - uint64_t start_timestamp; - - auto operator<=>(const SnapshotDurabilityInfo &) const = default; -}; - -/// Get list of snapshot files with their UUID. -/// @param snapshot_directory Directory containing the Snapshot files. -/// @param uuid UUID of the Snapshot files. If not empty, fetch only Snapshot -/// file with the specified UUID. Otherwise, fetch only Snapshot files in the -/// snapshot_directory. -/// @return List of snapshot files defined with its path and UUID. -std::vector GetSnapshotFiles(const std::filesystem::path &snapshot_directory, - std::string_view uuid = ""); - -/// Used to capture a WAL's data related to durability -struct WalDurabilityInfo { - explicit WalDurabilityInfo(const uint64_t seq_num, const uint64_t from_timestamp, const uint64_t to_timestamp, - std::string uuid, std::string epoch_id, std::filesystem::path path) - : seq_num(seq_num), - from_timestamp(from_timestamp), - to_timestamp(to_timestamp), - uuid(std::move(uuid)), - epoch_id(std::move(epoch_id)), - path(std::move(path)) {} - - uint64_t seq_num; - uint64_t from_timestamp; - uint64_t to_timestamp; - std::string uuid; - std::string epoch_id; - std::filesystem::path path; - - auto operator<=>(const WalDurabilityInfo &) const = default; -}; - -/// Get list of WAL files ordered by the sequence number -/// @param wal_directory Directory containing the WAL files. -/// @param uuid UUID of the WAL files. If not empty, fetch only WAL files -/// with the specified UUID. Otherwise, fetch all WAL files in the -/// wal_directory. -/// @param current_seq_num Sequence number of the WAL file which is currently -/// being written. If specified, load only finalized WAL files, i.e. WAL files -/// with seq_num < current_seq_num. -/// @return List of WAL files. Each WAL file is defined with its sequence -/// number, from timestamp, to timestamp and path. -std::optional> GetWalFiles(const std::filesystem::path &wal_directory, - std::string_view uuid = "", - std::optional current_seq_num = {}); - -// Helper function used to recover all discovered indices and constraints. The -// indices and constraints must be recovered after the data recovery is done -// to ensure that the indices and constraints are consistent at the end of the -// recovery process. -/// @throw RecoveryFailure -void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, - Constraints *constraints, VerticesSkipList *vertices); - -/// Recovers data either from a snapshot and/or WAL files. -/// @throw RecoveryFailure -/// @throw std::bad_alloc -std::optional RecoverData(const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, std::string *uuid, - std::string *epoch_id, - std::deque> *epoch_history, - VerticesSkipList *vertices, utils::SkipList *edges, uint64_t *edge_count, - NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, - Config::Items items, uint64_t *wal_seq_num); - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/exceptions.hpp b/src/storage/v3/durability/exceptions.hpp deleted file mode 100644 index 55f5e4e54..000000000 --- a/src/storage/v3/durability/exceptions.hpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include "utils/exceptions.hpp" - -namespace memgraph::storage::v3::durability { - -/// Exception used to handle errors during recovery. -class RecoveryFailure : public utils::BasicException { - using utils::BasicException::BasicException; -}; - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/marker.hpp b/src/storage/v3/durability/marker.hpp deleted file mode 100644 index da3ff2530..000000000 --- a/src/storage/v3/durability/marker.hpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include - -namespace memgraph::storage::v3::durability { - -/// Markers that are used to indicate crucial parts of the snapshot/WAL. -/// IMPORTANT: Don't forget to update the list of all markers `kMarkersAll` when -/// you add a new Marker. -enum class Marker : uint8_t { - TYPE_NULL = 0x10, - TYPE_BOOL = 0x11, - TYPE_INT = 0x12, - TYPE_DOUBLE = 0x13, - TYPE_STRING = 0x14, - TYPE_LIST = 0x15, - TYPE_MAP = 0x16, - TYPE_PROPERTY_VALUE = 0x17, - TYPE_TEMPORAL_DATA = 0x18, - - SECTION_VERTEX = 0x20, - SECTION_EDGE = 0x21, - SECTION_MAPPER = 0x22, - SECTION_METADATA = 0x23, - SECTION_INDICES = 0x24, - SECTION_CONSTRAINTS = 0x25, - SECTION_DELTA = 0x26, - SECTION_EPOCH_HISTORY = 0x27, - SECTION_OFFSETS = 0x42, - - DELTA_VERTEX_CREATE = 0x50, - DELTA_VERTEX_DELETE = 0x51, - DELTA_VERTEX_ADD_LABEL = 0x52, - DELTA_VERTEX_REMOVE_LABEL = 0x53, - DELTA_VERTEX_SET_PROPERTY = 0x54, - DELTA_EDGE_CREATE = 0x55, - DELTA_EDGE_DELETE = 0x56, - DELTA_EDGE_SET_PROPERTY = 0x57, - DELTA_TRANSACTION_END = 0x58, - DELTA_LABEL_INDEX_CREATE = 0x59, - DELTA_LABEL_INDEX_DROP = 0x5a, - DELTA_LABEL_PROPERTY_INDEX_CREATE = 0x5b, - DELTA_LABEL_PROPERTY_INDEX_DROP = 0x5c, - DELTA_EXISTENCE_CONSTRAINT_CREATE = 0x5d, - DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e, - DELTA_UNIQUE_CONSTRAINT_CREATE = 0x5f, - DELTA_UNIQUE_CONSTRAINT_DROP = 0x60, - - VALUE_FALSE = 0x00, - VALUE_TRUE = 0xff, -}; - -/// List of all available markers. -/// IMPORTANT: Don't forget to update this list when you add a new Marker. -static const Marker kMarkersAll[] = { - Marker::TYPE_NULL, - Marker::TYPE_BOOL, - Marker::TYPE_INT, - Marker::TYPE_DOUBLE, - Marker::TYPE_STRING, - Marker::TYPE_LIST, - Marker::TYPE_MAP, - Marker::TYPE_TEMPORAL_DATA, - Marker::TYPE_PROPERTY_VALUE, - Marker::SECTION_VERTEX, - Marker::SECTION_EDGE, - Marker::SECTION_MAPPER, - Marker::SECTION_METADATA, - Marker::SECTION_INDICES, - Marker::SECTION_CONSTRAINTS, - Marker::SECTION_DELTA, - Marker::SECTION_EPOCH_HISTORY, - Marker::SECTION_OFFSETS, - Marker::DELTA_VERTEX_CREATE, - Marker::DELTA_VERTEX_DELETE, - Marker::DELTA_VERTEX_ADD_LABEL, - Marker::DELTA_VERTEX_REMOVE_LABEL, - Marker::DELTA_VERTEX_SET_PROPERTY, - Marker::DELTA_EDGE_CREATE, - Marker::DELTA_EDGE_DELETE, - Marker::DELTA_EDGE_SET_PROPERTY, - Marker::DELTA_TRANSACTION_END, - Marker::DELTA_LABEL_INDEX_CREATE, - Marker::DELTA_LABEL_INDEX_DROP, - Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE, - Marker::DELTA_LABEL_PROPERTY_INDEX_DROP, - Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE, - Marker::DELTA_EXISTENCE_CONSTRAINT_DROP, - Marker::DELTA_UNIQUE_CONSTRAINT_CREATE, - Marker::DELTA_UNIQUE_CONSTRAINT_DROP, - Marker::VALUE_FALSE, - Marker::VALUE_TRUE, -}; - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/metadata.hpp b/src/storage/v3/durability/metadata.hpp deleted file mode 100644 index d6377d278..000000000 --- a/src/storage/v3/durability/metadata.hpp +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include -#include - -#include "storage/v3/durability/exceptions.hpp" -#include "storage/v3/id_types.hpp" - -namespace memgraph::storage::v3::durability { - -/// Structure used to hold metadata about the recovered snapshot/WAL. -struct RecoveryInfo { - uint64_t next_vertex_id{0}; - uint64_t next_edge_id{0}; - uint64_t next_timestamp{0}; - - // last timestamp read from a WAL file - std::optional last_commit_timestamp; -}; - -/// Structure used to track indices and constraints during recovery. -struct RecoveredIndicesAndConstraints { - struct { - std::vector label; - std::vector> label_property; - } indices; - - struct { - std::vector> existence; - std::vector>> unique; - } constraints; -}; - -// Helper function used to insert indices/constraints into the recovered -// indices/constraints object. -// @throw RecoveryFailure -template -void AddRecoveredIndexConstraint(std::vector *list, TObj obj, const char *error_message) { - auto it = std::find(list->begin(), list->end(), obj); - if (it == list->end()) { - list->push_back(obj); - } else { - throw RecoveryFailure(error_message); - } -} - -// Helper function used to remove indices/constraints from the recovered -// indices/constraints object. -// @throw RecoveryFailure -template -void RemoveRecoveredIndexConstraint(std::vector *list, TObj obj, const char *error_message) { - auto it = std::find(list->begin(), list->end(), obj); - if (it != list->end()) { - std::swap(*it, list->back()); - list->pop_back(); - } else { - throw RecoveryFailure(error_message); - } -} - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/paths.hpp b/src/storage/v3/durability/paths.hpp deleted file mode 100644 index 1b9ecd1f9..000000000 --- a/src/storage/v3/durability/paths.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include - -#include "utils/timestamp.hpp" - -namespace memgraph::storage::v3::durability { - -static const std::string kSnapshotDirectory{"snapshots"}; -static const std::string kWalDirectory{"wal"}; -static const std::string kBackupDirectory{".backup"}; -static const std::string kLockFile{".lock"}; - -// This is the prefix used for Snapshot and WAL filenames. It is a timestamp -// format that equals to: YYYYmmddHHMMSSffffff -const std::string kTimestampFormat = "{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}{:06d}"; - -// Generates the name for a snapshot in a well-defined sortable format with the -// start timestamp appended to the file name. -inline std::string MakeSnapshotName(uint64_t start_timestamp) { - std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat); - return date_str + "_timestamp_" + std::to_string(start_timestamp); -} - -// Generates the name for a WAL file in a well-defined sortable format. -inline std::string MakeWalName() { - std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat); - return date_str + "_current"; -} - -// Generates the name for a WAL file in a well-defined sortable format with the -// range of timestamps contained [from, to] appended to the name. -inline std::string RemakeWalName(const std::string ¤t_name, uint64_t from_timestamp, uint64_t to_timestamp) { - return current_name.substr(0, current_name.size() - 8) + "_from_" + std::to_string(from_timestamp) + "_to_" + - std::to_string(to_timestamp); -} - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/serialization.cpp b/src/storage/v3/durability/serialization.cpp deleted file mode 100644 index 69b9c2206..000000000 --- a/src/storage/v3/durability/serialization.cpp +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/durability/serialization.hpp" - -#include "storage/v3/temporal.hpp" -#include "utils/endian.hpp" - -namespace memgraph::storage::v3::durability { - -////////////////////////// -// Encoder implementation. -////////////////////////// - -namespace { -void WriteSize(Encoder *encoder, uint64_t size) { - size = utils::HostToLittleEndian(size); - encoder->Write(reinterpret_cast(&size), sizeof(size)); -} -} // namespace - -void Encoder::Initialize(const std::filesystem::path &path, const std::string_view &magic, uint64_t version) { - file_.Open(path, utils::OutputFile::Mode::OVERWRITE_EXISTING); - Write(reinterpret_cast(magic.data()), magic.size()); - auto version_encoded = utils::HostToLittleEndian(version); - Write(reinterpret_cast(&version_encoded), sizeof(version_encoded)); -} - -void Encoder::OpenExisting(const std::filesystem::path &path) { - file_.Open(path, utils::OutputFile::Mode::APPEND_TO_EXISTING); -} - -void Encoder::Close() { - if (file_.IsOpen()) { - file_.Close(); - } -} - -void Encoder::Write(const uint8_t *data, uint64_t size) { file_.Write(data, size); } - -void Encoder::WriteMarker(Marker marker) { - auto value = static_cast(marker); - Write(&value, sizeof(value)); -} - -void Encoder::WriteBool(bool value) { - WriteMarker(Marker::TYPE_BOOL); - if (value) { - WriteMarker(Marker::VALUE_TRUE); - } else { - WriteMarker(Marker::VALUE_FALSE); - } -} - -void Encoder::WriteUint(uint64_t value) { - value = utils::HostToLittleEndian(value); - WriteMarker(Marker::TYPE_INT); - Write(reinterpret_cast(&value), sizeof(value)); -} - -void Encoder::WriteDouble(double value) { - auto value_uint = utils::MemcpyCast(value); - value_uint = utils::HostToLittleEndian(value_uint); - WriteMarker(Marker::TYPE_DOUBLE); - Write(reinterpret_cast(&value_uint), sizeof(value_uint)); -} - -void Encoder::WriteString(const std::string_view &value) { - WriteMarker(Marker::TYPE_STRING); - WriteSize(this, value.size()); - Write(reinterpret_cast(value.data()), value.size()); -} - -void Encoder::WritePropertyValue(const PropertyValue &value) { - WriteMarker(Marker::TYPE_PROPERTY_VALUE); - switch (value.type()) { - case PropertyValue::Type::Null: { - WriteMarker(Marker::TYPE_NULL); - break; - } - case PropertyValue::Type::Bool: { - WriteBool(value.ValueBool()); - break; - } - case PropertyValue::Type::Int: { - WriteUint(utils::MemcpyCast(value.ValueInt())); - break; - } - case PropertyValue::Type::Double: { - WriteDouble(value.ValueDouble()); - break; - } - case PropertyValue::Type::String: { - WriteString(value.ValueString()); - break; - } - case PropertyValue::Type::List: { - const auto &list = value.ValueList(); - WriteMarker(Marker::TYPE_LIST); - WriteSize(this, list.size()); - for (const auto &item : list) { - WritePropertyValue(item); - } - break; - } - case PropertyValue::Type::Map: { - const auto &map = value.ValueMap(); - WriteMarker(Marker::TYPE_MAP); - WriteSize(this, map.size()); - for (const auto &item : map) { - WriteString(item.first); - WritePropertyValue(item.second); - } - break; - } - case PropertyValue::Type::TemporalData: { - const auto temporal_data = value.ValueTemporalData(); - WriteMarker(Marker::TYPE_TEMPORAL_DATA); - WriteUint(static_cast(temporal_data.type)); - WriteUint(utils::MemcpyCast(temporal_data.microseconds)); - break; - } - } -} - -uint64_t Encoder::GetPosition() { return file_.GetPosition(); } - -void Encoder::SetPosition(uint64_t position) { - file_.SetPosition(utils::OutputFile::Position::SET, static_cast(position)); -} - -void Encoder::Sync() { file_.Sync(); } - -void Encoder::Finalize() { - file_.Sync(); - file_.Close(); -} - -void Encoder::DisableFlushing() { file_.DisableFlushing(); } - -void Encoder::EnableFlushing() { file_.EnableFlushing(); } - -void Encoder::TryFlushing() { file_.TryFlushing(); } - -std::pair Encoder::CurrentFileBuffer() const { return file_.CurrentBuffer(); } - -size_t Encoder::GetSize() { return file_.GetSize(); } - -////////////////////////// -// Decoder implementation. -////////////////////////// - -namespace { -std::optional CastToMarker(uint8_t value) { - for (auto marker : kMarkersAll) { - if (static_cast(marker) == value) { - return marker; - } - } - return std::nullopt; -} - -std::optional ReadSize(Decoder *decoder) { - uint64_t size{0}; - if (!decoder->Read(reinterpret_cast(&size), sizeof(size))) return std::nullopt; - size = utils::LittleEndianToHost(size); - return size; -} -} // namespace - -std::optional Decoder::Initialize(const std::filesystem::path &path, const std::string &magic) { - if (!file_.Open(path)) return std::nullopt; - std::string file_magic(magic.size(), '\0'); - if (!Read(reinterpret_cast(file_magic.data()), file_magic.size())) return std::nullopt; - if (file_magic != magic) return std::nullopt; - uint64_t version_encoded{0}; - if (!Read(reinterpret_cast(&version_encoded), sizeof(version_encoded))) return std::nullopt; - return utils::LittleEndianToHost(version_encoded); -} - -bool Decoder::Read(uint8_t *data, size_t size) { return file_.Read(data, size); } - -bool Decoder::Peek(uint8_t *data, size_t size) { return file_.Peek(data, size); } - -std::optional Decoder::PeekMarker() { - uint8_t value{0}; - if (!Peek(&value, sizeof(value))) return std::nullopt; - auto marker = CastToMarker(value); - if (!marker) return std::nullopt; - return *marker; -} - -std::optional Decoder::ReadMarker() { - uint8_t value{0}; - if (!Read(&value, sizeof(value))) return std::nullopt; - auto marker = CastToMarker(value); - if (!marker) return std::nullopt; - return *marker; -} - -std::optional Decoder::ReadBool() { - auto marker = ReadMarker(); - if (!marker || *marker != Marker::TYPE_BOOL) return std::nullopt; - auto value = ReadMarker(); - if (!value || (*value != Marker::VALUE_FALSE && *value != Marker::VALUE_TRUE)) return std::nullopt; - return *value == Marker::VALUE_TRUE; -} - -std::optional Decoder::ReadUint() { - auto marker = ReadMarker(); - if (!marker || *marker != Marker::TYPE_INT) return std::nullopt; - uint64_t value{0}; - if (!Read(reinterpret_cast(&value), sizeof(value))) return std::nullopt; - value = utils::LittleEndianToHost(value); - return value; -} - -std::optional Decoder::ReadDouble() { - auto marker = ReadMarker(); - if (!marker || *marker != Marker::TYPE_DOUBLE) return std::nullopt; - uint64_t value_int{0}; - if (!Read(reinterpret_cast(&value_int), sizeof(value_int))) return std::nullopt; - value_int = utils::LittleEndianToHost(value_int); - auto value = utils::MemcpyCast(value_int); - return value; -} - -std::optional Decoder::ReadString() { - auto marker = ReadMarker(); - if (!marker || *marker != Marker::TYPE_STRING) return std::nullopt; - auto size = ReadSize(this); - if (!size) return std::nullopt; - std::string value(*size, '\0'); - if (!Read(reinterpret_cast(value.data()), *size)) return std::nullopt; - return value; -} - -namespace { -std::optional ReadTemporalData(Decoder &decoder) { - const auto inner_marker = decoder.ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_TEMPORAL_DATA) return std::nullopt; - - const auto type = decoder.ReadUint(); - if (!type) return std::nullopt; - - const auto microseconds = decoder.ReadUint(); - if (!microseconds) return std::nullopt; - - return TemporalData{static_cast(*type), utils::MemcpyCast(*microseconds)}; -} -} // namespace - -std::optional Decoder::ReadPropertyValue() { - auto pv_marker = ReadMarker(); - if (!pv_marker || *pv_marker != Marker::TYPE_PROPERTY_VALUE) return std::nullopt; - - auto marker = PeekMarker(); - if (!marker) return std::nullopt; - switch (*marker) { - case Marker::TYPE_NULL: { - auto inner_marker = ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_NULL) return std::nullopt; - return PropertyValue(); - } - case Marker::TYPE_BOOL: { - auto value = ReadBool(); - if (!value) return std::nullopt; - return PropertyValue(*value); - } - case Marker::TYPE_INT: { - auto value = ReadUint(); - if (!value) return std::nullopt; - return PropertyValue(utils::MemcpyCast(*value)); - } - case Marker::TYPE_DOUBLE: { - auto value = ReadDouble(); - if (!value) return std::nullopt; - return PropertyValue(*value); - } - case Marker::TYPE_STRING: { - auto value = ReadString(); - if (!value) return std::nullopt; - return PropertyValue(std::move(*value)); - } - case Marker::TYPE_LIST: { - auto inner_marker = ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_LIST) return std::nullopt; - auto size = ReadSize(this); - if (!size) return std::nullopt; - std::vector value; - value.reserve(*size); - for (uint64_t i = 0; i < *size; ++i) { - auto item = ReadPropertyValue(); - if (!item) return std::nullopt; - value.emplace_back(std::move(*item)); - } - return PropertyValue(std::move(value)); - } - case Marker::TYPE_MAP: { - auto inner_marker = ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_MAP) return std::nullopt; - auto size = ReadSize(this); - if (!size) return std::nullopt; - std::map value; - for (uint64_t i = 0; i < *size; ++i) { - auto key = ReadString(); - if (!key) return std::nullopt; - auto item = ReadPropertyValue(); - if (!item) return std::nullopt; - value.emplace(std::move(*key), std::move(*item)); - } - return PropertyValue(std::move(value)); - } - case Marker::TYPE_TEMPORAL_DATA: { - const auto maybe_temporal_data = ReadTemporalData(*this); - if (!maybe_temporal_data) return std::nullopt; - return PropertyValue(*maybe_temporal_data); - } - - case Marker::TYPE_PROPERTY_VALUE: - case Marker::SECTION_VERTEX: - case Marker::SECTION_EDGE: - case Marker::SECTION_MAPPER: - case Marker::SECTION_METADATA: - case Marker::SECTION_INDICES: - case Marker::SECTION_CONSTRAINTS: - case Marker::SECTION_DELTA: - case Marker::SECTION_EPOCH_HISTORY: - case Marker::SECTION_OFFSETS: - case Marker::DELTA_VERTEX_CREATE: - case Marker::DELTA_VERTEX_DELETE: - case Marker::DELTA_VERTEX_ADD_LABEL: - case Marker::DELTA_VERTEX_REMOVE_LABEL: - case Marker::DELTA_VERTEX_SET_PROPERTY: - case Marker::DELTA_EDGE_CREATE: - case Marker::DELTA_EDGE_DELETE: - case Marker::DELTA_EDGE_SET_PROPERTY: - case Marker::DELTA_TRANSACTION_END: - case Marker::DELTA_LABEL_INDEX_CREATE: - case Marker::DELTA_LABEL_INDEX_DROP: - case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: - case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: - case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: - case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: - case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE: - case Marker::DELTA_UNIQUE_CONSTRAINT_DROP: - case Marker::VALUE_FALSE: - case Marker::VALUE_TRUE: - return std::nullopt; - } -} - -bool Decoder::SkipString() { - auto marker = ReadMarker(); - if (!marker || *marker != Marker::TYPE_STRING) return false; - auto maybe_size = ReadSize(this); - if (!maybe_size) return false; - - const uint64_t kBufferSize = 262144; - uint8_t buffer[kBufferSize]; - uint64_t size = *maybe_size; - while (size > 0) { - uint64_t to_read = size < kBufferSize ? size : kBufferSize; - if (!Read(reinterpret_cast(&buffer), to_read)) return false; - size -= to_read; - } - - return true; -} - -bool Decoder::SkipPropertyValue() { - auto pv_marker = ReadMarker(); - if (!pv_marker || *pv_marker != Marker::TYPE_PROPERTY_VALUE) return false; - - auto marker = PeekMarker(); - if (!marker) return false; - switch (*marker) { - case Marker::TYPE_NULL: { - auto inner_marker = ReadMarker(); - return inner_marker && *inner_marker == Marker::TYPE_NULL; - } - case Marker::TYPE_BOOL: { - return !!ReadBool(); - } - case Marker::TYPE_INT: { - return !!ReadUint(); - } - case Marker::TYPE_DOUBLE: { - return !!ReadDouble(); - } - case Marker::TYPE_STRING: { - return SkipString(); - } - case Marker::TYPE_LIST: { - auto inner_marker = ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_LIST) return false; - auto size = ReadSize(this); - if (!size) return false; - for (uint64_t i = 0; i < *size; ++i) { - if (!SkipPropertyValue()) return false; - } - return true; - } - case Marker::TYPE_MAP: { - auto inner_marker = ReadMarker(); - if (!inner_marker || *inner_marker != Marker::TYPE_MAP) return false; - auto size = ReadSize(this); - if (!size) return false; - for (uint64_t i = 0; i < *size; ++i) { - if (!SkipString()) return false; - if (!SkipPropertyValue()) return false; - } - return true; - } - case Marker::TYPE_TEMPORAL_DATA: { - return !!ReadTemporalData(*this); - } - - case Marker::TYPE_PROPERTY_VALUE: - case Marker::SECTION_VERTEX: - case Marker::SECTION_EDGE: - case Marker::SECTION_MAPPER: - case Marker::SECTION_METADATA: - case Marker::SECTION_INDICES: - case Marker::SECTION_CONSTRAINTS: - case Marker::SECTION_DELTA: - case Marker::SECTION_EPOCH_HISTORY: - case Marker::SECTION_OFFSETS: - case Marker::DELTA_VERTEX_CREATE: - case Marker::DELTA_VERTEX_DELETE: - case Marker::DELTA_VERTEX_ADD_LABEL: - case Marker::DELTA_VERTEX_REMOVE_LABEL: - case Marker::DELTA_VERTEX_SET_PROPERTY: - case Marker::DELTA_EDGE_CREATE: - case Marker::DELTA_EDGE_DELETE: - case Marker::DELTA_EDGE_SET_PROPERTY: - case Marker::DELTA_TRANSACTION_END: - case Marker::DELTA_LABEL_INDEX_CREATE: - case Marker::DELTA_LABEL_INDEX_DROP: - case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: - case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: - case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: - case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: - case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE: - case Marker::DELTA_UNIQUE_CONSTRAINT_DROP: - case Marker::VALUE_FALSE: - case Marker::VALUE_TRUE: - return false; - } -} - -std::optional Decoder::GetSize() { return file_.GetSize(); } - -std::optional Decoder::GetPosition() { return file_.GetPosition(); } - -bool Decoder::SetPosition(uint64_t position) { - return !!file_.SetPosition(utils::InputFile::Position::SET, static_cast(position)); -} - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/serialization.hpp b/src/storage/v3/durability/serialization.hpp deleted file mode 100644 index 79908fcd3..000000000 --- a/src/storage/v3/durability/serialization.hpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -#include "storage/v3/config.hpp" -#include "storage/v3/durability/marker.hpp" -#include "storage/v3/name_id_mapper.hpp" -#include "storage/v3/property_value.hpp" -#include "utils/file.hpp" - -namespace memgraph::storage::v3::durability { - -/// Encoder interface class. Used to implement streams to different targets -/// (e.g. file and network). -class BaseEncoder { - protected: - BaseEncoder() = default; - ~BaseEncoder() = default; - - public: - BaseEncoder(const BaseEncoder &) = delete; - BaseEncoder(BaseEncoder &&) = delete; - BaseEncoder &operator=(const BaseEncoder &) = delete; - BaseEncoder &operator=(BaseEncoder &&) = delete; - - virtual void WriteMarker(Marker marker) = 0; - virtual void WriteBool(bool value) = 0; - virtual void WriteUint(uint64_t value) = 0; - virtual void WriteDouble(double value) = 0; - virtual void WriteString(const std::string_view &value) = 0; - virtual void WritePropertyValue(const PropertyValue &value) = 0; -}; - -/// Encoder that is used to generate a snapshot/WAL. -class Encoder final : public BaseEncoder { - public: - void Initialize(const std::filesystem::path &path, const std::string_view &magic, uint64_t version); - - void OpenExisting(const std::filesystem::path &path); - - void Close(); - // Main write function, the only one that is allowed to write to the `file_` - // directly. - void Write(const uint8_t *data, uint64_t size); - - void WriteMarker(Marker marker) override; - void WriteBool(bool value) override; - void WriteUint(uint64_t value) override; - void WriteDouble(double value) override; - void WriteString(const std::string_view &value) override; - void WritePropertyValue(const PropertyValue &value) override; - - uint64_t GetPosition(); - void SetPosition(uint64_t position); - - void Sync(); - - void Finalize(); - - // Disable flushing of the internal buffer. - void DisableFlushing(); - // Enable flushing of the internal buffer. - void EnableFlushing(); - // Try flushing the internal buffer. - void TryFlushing(); - // Get the current internal buffer with its size. - std::pair CurrentFileBuffer() const; - - // Get the total size of the current file. - size_t GetSize(); - - private: - utils::OutputFile file_; -}; - -/// Decoder interface class. Used to implement streams from different sources -/// (e.g. file and network). -class BaseDecoder { - protected: - BaseDecoder() = default; - ~BaseDecoder() = default; - - public: - BaseDecoder(const BaseDecoder &) = delete; - BaseDecoder(BaseDecoder &&) = delete; - BaseDecoder &operator=(const BaseDecoder &) = delete; - BaseDecoder &operator=(BaseDecoder &&) = delete; - - virtual std::optional ReadMarker() = 0; - virtual std::optional ReadBool() = 0; - virtual std::optional ReadUint() = 0; - virtual std::optional ReadDouble() = 0; - virtual std::optional ReadString() = 0; - virtual std::optional ReadPropertyValue() = 0; - - virtual bool SkipString() = 0; - virtual bool SkipPropertyValue() = 0; -}; - -/// Decoder that is used to read a generated snapshot/WAL. -class Decoder final : public BaseDecoder { - public: - std::optional Initialize(const std::filesystem::path &path, const std::string &magic); - - // Main read functions, the only one that are allowed to read from the `file_` - // directly. - bool Read(uint8_t *data, size_t size); - bool Peek(uint8_t *data, size_t size); - - std::optional PeekMarker(); - - std::optional ReadMarker() override; - std::optional ReadBool() override; - std::optional ReadUint() override; - std::optional ReadDouble() override; - std::optional ReadString() override; - std::optional ReadPropertyValue() override; - - bool SkipString() override; - bool SkipPropertyValue() override; - - std::optional GetSize(); - std::optional GetPosition(); - bool SetPosition(uint64_t position); - - private: - utils::InputFile file_; -}; - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/snapshot.cpp b/src/storage/v3/durability/snapshot.cpp deleted file mode 100644 index f7ae7b352..000000000 --- a/src/storage/v3/durability/snapshot.cpp +++ /dev/null @@ -1,1002 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/durability/snapshot.hpp" - -#include "storage/v3/durability/exceptions.hpp" -#include "storage/v3/durability/paths.hpp" -#include "storage/v3/durability/serialization.hpp" -#include "storage/v3/durability/version.hpp" -#include "storage/v3/durability/wal.hpp" -#include "storage/v3/edge_accessor.hpp" -#include "storage/v3/edge_ref.hpp" -#include "storage/v3/mvcc.hpp" -#include "storage/v3/schema_validator.hpp" -#include "storage/v3/schemas.hpp" -#include "storage/v3/vertex_accessor.hpp" -#include "storage/v3/vertex_id.hpp" -#include "storage/v3/vertices_skip_list.hpp" -#include "utils/file_locker.hpp" -#include "utils/logging.hpp" -#include "utils/message.hpp" - -namespace memgraph::storage::v3::durability { - -// Snapshot format: -// -// 1) Magic string (non-encoded) -// -// 2) Snapshot version (non-encoded, little-endian) -// -// 3) Section offsets: -// * offset to the first edge in the snapshot (`0` if properties on edges -// are disabled) -// * offset to the first vertex in the snapshot -// * offset to the indices section -// * offset to the constraints section -// * offset to the mapper section -// * offset to the metadata section -// -// 4) Encoded edges (if properties on edges are enabled); each edge is written -// in the following format: -// * gid -// * properties -// -// 5) Encoded vertices; each vertex is written in the following format: -// * gid -// * labels -// * properties -// * in edges -// * edge gid -// * from vertex gid -// * edge type -// * out edges -// * edge gid -// * to vertex gid -// * edge type -// -// 6) Indices -// * label indices -// * label -// * label+property indices -// * label -// * property -// -// 7) Constraints -// * existence constraints -// * label -// * property -// * unique constraints (from version 13) -// * label -// * properties -// -// 8) Name to ID mapper data -// * id to name mappings -// * id -// * name -// -// 9) Metadata -// * storage UUID -// * snapshot transaction start timestamp (required when recovering -// from snapshot combined with WAL to determine what deltas need to be -// applied) -// * number of edges -// * number of vertices -// -// IMPORTANT: When changing snapshot encoding/decoding bump the snapshot/WAL -// version in `version.hpp`. - -namespace { -constexpr auto kDummyLabelId = LabelId::FromUint(0); -} // namespace - -// Function used to read information about the snapshot file. -SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) { - // Check magic and version. - Decoder snapshot; - auto version = snapshot.Initialize(path, kSnapshotMagic); - if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!"); - if (!IsVersionSupported(*version)) throw RecoveryFailure("Invalid snapshot version!"); - - // Prepare return value. - SnapshotInfo info; - - // Read offsets. - { - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_OFFSETS) throw RecoveryFailure("Invalid snapshot data!"); - - auto snapshot_size = snapshot.GetSize(); - if (!snapshot_size) throw RecoveryFailure("Couldn't read data from snapshot!"); - - auto read_offset = [&snapshot, snapshot_size] { - auto maybe_offset = snapshot.ReadUint(); - if (!maybe_offset) throw RecoveryFailure("Invalid snapshot format!"); - auto offset = *maybe_offset; - if (offset > *snapshot_size) throw RecoveryFailure("Invalid snapshot format!"); - return offset; - }; - - info.offset_edges = read_offset(); - info.offset_vertices = read_offset(); - info.offset_indices = read_offset(); - info.offset_constraints = read_offset(); - info.offset_mapper = read_offset(); - info.offset_epoch_history = read_offset(); - info.offset_metadata = read_offset(); - } - - // Read metadata. - { - if (!snapshot.SetPosition(info.offset_metadata)) throw RecoveryFailure("Couldn't read data from snapshot!"); - - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_METADATA) throw RecoveryFailure("Invalid snapshot data!"); - - auto maybe_uuid = snapshot.ReadString(); - if (!maybe_uuid) throw RecoveryFailure("Invalid snapshot data!"); - info.uuid = std::move(*maybe_uuid); - - auto maybe_epoch_id = snapshot.ReadString(); - if (!maybe_epoch_id) throw RecoveryFailure("Invalid snapshot data!"); - info.epoch_id = std::move(*maybe_epoch_id); - - auto maybe_timestamp = snapshot.ReadUint(); - if (!maybe_timestamp) throw RecoveryFailure("Invalid snapshot data!"); - info.start_timestamp = *maybe_timestamp; - - auto maybe_edges = snapshot.ReadUint(); - if (!maybe_edges) throw RecoveryFailure("Invalid snapshot data!"); - info.edges_count = *maybe_edges; - - auto maybe_vertices = snapshot.ReadUint(); - if (!maybe_vertices) throw RecoveryFailure("Invalid snapshot data!"); - info.vertices_count = *maybe_vertices; - } - - return info; -} - -RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices, - utils::SkipList *edges, - std::deque> *epoch_history, - NameIdMapper *name_id_mapper, uint64_t *edge_count, Config::Items items) { - RecoveryInfo ret; - RecoveredIndicesAndConstraints indices_constraints; - - Decoder snapshot; - auto version = snapshot.Initialize(path, kSnapshotMagic); - if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!"); - if (!IsVersionSupported(*version)) throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version)); - - // Cleanup of loaded data in case of failure. - bool success = false; - utils::OnScopeExit cleanup([&] { - if (!success) { - edges->clear(); - vertices->clear(); - epoch_history->clear(); - } - }); - - // Read snapshot info. - const auto info = ReadSnapshotInfo(path); - spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count); - // Check for edges. - bool snapshot_has_edges = info.offset_edges != 0; - - // Recover mapper. - std::unordered_map snapshot_id_map; - { - spdlog::info("Recovering mapper metadata."); - if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!"); - - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_MAPPER) throw RecoveryFailure("Invalid snapshot data!"); - - auto size = snapshot.ReadUint(); - if (!size) throw RecoveryFailure("Invalid snapshot data!"); - - for (uint64_t i = 0; i < *size; ++i) { - auto id = snapshot.ReadUint(); - if (!id) throw RecoveryFailure("Invalid snapshot data!"); - auto name = snapshot.ReadString(); - if (!name) throw RecoveryFailure("Invalid snapshot data!"); - auto my_id = name_id_mapper->NameToId(*name); - snapshot_id_map.emplace(*id, my_id); - SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id); - } - } - auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) { - auto it = snapshot_id_map.find(snapshot_id); - if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); - return LabelId::FromUint(it->second); - }; - auto get_property_from_id = [&snapshot_id_map](uint64_t snapshot_id) { - auto it = snapshot_id_map.find(snapshot_id); - if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); - return PropertyId::FromUint(it->second); - }; - auto get_edge_type_from_id = [&snapshot_id_map](uint64_t snapshot_id) { - auto it = snapshot_id_map.find(snapshot_id); - if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); - return EdgeTypeId::FromUint(it->second); - }; - - // Reset current edge count. - *edge_count = 0; - - { - // Recover edges. - auto edge_acc = edges->access(); - uint64_t last_edge_gid = 0; - if (snapshot_has_edges) { - spdlog::info("Recovering {} edges.", info.edges_count); - if (!snapshot.SetPosition(info.offset_edges)) throw RecoveryFailure("Couldn't read data from snapshot!"); - for (uint64_t i = 0; i < info.edges_count; ++i) { - { - const auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_EDGE) throw RecoveryFailure("Invalid snapshot data!"); - } - - if (items.properties_on_edges) { - // Insert edge. - auto gid = snapshot.ReadUint(); - if (!gid) throw RecoveryFailure("Invalid snapshot data!"); - if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - last_edge_gid = *gid; - spdlog::debug("Recovering edge {} with properties.", *gid); - auto [it, inserted] = edge_acc.insert(Edge{Gid::FromUint(*gid), nullptr}); - if (!inserted) throw RecoveryFailure("The edge must be inserted here!"); - - // Recover properties. - { - auto props_size = snapshot.ReadUint(); - if (!props_size) throw RecoveryFailure("Invalid snapshot data!"); - auto &props = it->properties; - for (uint64_t j = 0; j < *props_size; ++j) { - auto key = snapshot.ReadUint(); - if (!key) throw RecoveryFailure("Invalid snapshot data!"); - auto value = snapshot.ReadPropertyValue(); - if (!value) throw RecoveryFailure("Invalid snapshot data!"); - SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for edge {}.", - name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid); - props.SetProperty(get_property_from_id(*key), *value); - } - } - } else { - // Read edge GID. - auto gid = snapshot.ReadUint(); - if (!gid) throw RecoveryFailure("Invalid snapshot data!"); - if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - last_edge_gid = *gid; - - spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid); - // Read properties. - { - auto props_size = snapshot.ReadUint(); - if (!props_size) throw RecoveryFailure("Invalid snapshot data!"); - if (*props_size != 0) - throw RecoveryFailure( - "The snapshot has properties on edges, but the storage is " - "configured without properties on edges!"); - } - } - } - spdlog::info("Edges are recovered."); - } - - // Recover vertices (labels and properties). - if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!"); - auto vertex_acc = vertices->access(); - uint64_t last_vertex_gid = 0; - spdlog::info("Recovering {} vertices.", info.vertices_count); - for (uint64_t i = 0; i < info.vertices_count; ++i) { - { - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!"); - } - - // Insert vertex. - // auto gid = snapshot.ReadUint(); - // if (!gid) throw RecoveryFailure("Invalid snapshot data!"); - // if (i > 0 && *gid <= last_vertex_gid) { - // throw RecoveryFailure("Invalid snapshot data!"); - // } - // last_vertex_gid = *gid; - // spdlog::debug("Recovering vertex {}.", *gid); - // auto [it, inserted] = vertex_acc.insert({Vertex{Gid::FromUint(*gid), nullptr}}); - // if (!inserted) throw RecoveryFailure("The vertex must be inserted here!"); - - // Recover labels. - // spdlog::trace("Recovering labels for vertex {}.", *gid); - // { - // auto labels_size = snapshot.ReadUint(); - // if (!labels_size) throw RecoveryFailure("Invalid snapshot data!"); - // auto &labels = it->vertex.labels; - // labels.reserve(*labels_size); - // for (uint64_t j = 0; j < *labels_size; ++j) { - // auto label = snapshot.ReadUint(); - // if (!label) throw RecoveryFailure("Invalid snapshot data!"); - // SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)), - // *gid); - // labels.emplace_back(get_label_from_id(*label)); - // } - // } - - // Recover properties. - // spdlog::trace("Recovering properties for vertex {}.", *gid); - // { - // auto props_size = snapshot.ReadUint(); - // if (!props_size) throw RecoveryFailure("Invalid snapshot data!"); - // auto &props = it->vertex.properties; - // for (uint64_t j = 0; j < *props_size; ++j) { - // auto key = snapshot.ReadUint(); - // if (!key) throw RecoveryFailure("Invalid snapshot data!"); - // auto value = snapshot.ReadPropertyValue(); - // if (!value) throw RecoveryFailure("Invalid snapshot data!"); - // SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.", - // name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid); - // props.SetProperty(get_property_from_id(*key), *value); - // } - // } - - // Skip in edges. - { - auto in_size = snapshot.ReadUint(); - if (!in_size) throw RecoveryFailure("Invalid snapshot data!"); - for (uint64_t j = 0; j < *in_size; ++j) { - auto edge_gid = snapshot.ReadUint(); - if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto from_gid = snapshot.ReadUint(); - if (!from_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto edge_type = snapshot.ReadUint(); - if (!edge_type) throw RecoveryFailure("Invalid snapshot data!"); - } - } - - // Skip out edges. - auto out_size = snapshot.ReadUint(); - if (!out_size) throw RecoveryFailure("Invalid snapshot data!"); - for (uint64_t j = 0; j < *out_size; ++j) { - auto edge_gid = snapshot.ReadUint(); - if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto to_gid = snapshot.ReadUint(); - if (!to_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto edge_type = snapshot.ReadUint(); - if (!edge_type) throw RecoveryFailure("Invalid snapshot data!"); - } - } - spdlog::info("Vertices are recovered."); - - // Recover vertices (in/out edges). - spdlog::info("Recovering connectivity."); - if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!"); - for (auto &lgo_vertex : vertex_acc) { - auto &vertex = lgo_vertex.vertex; - { - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!"); - } - - // spdlog::trace("Recovering connectivity for vertex {}.", vertex.Gid().AsUint()); - // Check vertex. - // auto gid = snapshot.ReadUint(); - // if (!gid) throw RecoveryFailure("Invalid snapshot data!"); - // if (gid != vertex.Gid().AsUint()) throw RecoveryFailure("Invalid snapshot data!"); - - // Skip labels. - { - auto labels_size = snapshot.ReadUint(); - if (!labels_size) throw RecoveryFailure("Invalid snapshot data!"); - for (uint64_t j = 0; j < *labels_size; ++j) { - auto label = snapshot.ReadUint(); - if (!label) throw RecoveryFailure("Invalid snapshot data!"); - } - } - - // Skip properties. - { - auto props_size = snapshot.ReadUint(); - if (!props_size) throw RecoveryFailure("Invalid snapshot data!"); - for (uint64_t j = 0; j < *props_size; ++j) { - auto key = snapshot.ReadUint(); - if (!key) throw RecoveryFailure("Invalid snapshot data!"); - auto value = snapshot.SkipPropertyValue(); - if (!value) throw RecoveryFailure("Invalid snapshot data!"); - } - } - - // Recover in edges. - { - // TODO Fix Gid - spdlog::trace("Recovering inbound edges for vertex {}.", 1); - auto in_size = snapshot.ReadUint(); - if (!in_size) throw RecoveryFailure("Invalid snapshot data!"); - vertex.in_edges.reserve(*in_size); - for (uint64_t j = 0; j < *in_size; ++j) { - auto edge_gid = snapshot.ReadUint(); - if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - last_edge_gid = std::max(last_edge_gid, *edge_gid); - - auto from_gid = snapshot.ReadUint(); - if (!from_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto edge_type = snapshot.ReadUint(); - if (!edge_type) throw RecoveryFailure("Invalid snapshot data!"); - - auto from_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*from_gid).AsInt()}}); - if (from_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid from vertex!"); - - EdgeRef edge_ref(Gid::FromUint(*edge_gid)); - if (items.properties_on_edges) { - if (snapshot_has_edges) { - auto edge = edge_acc.find(Gid::FromUint(*edge_gid)); - if (edge == edge_acc.end()) throw RecoveryFailure("Invalid edge!"); - edge_ref = EdgeRef(&*edge); - } else { - auto [edge, inserted] = edge_acc.insert(Edge{Gid::FromUint(*edge_gid), nullptr}); - edge_ref = EdgeRef(&*edge); - } - } - // TODO Fix Gid - SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid, - name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), 1); - vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), VertexId{kDummyLabelId, {}}, edge_ref); - } - } - - // Recover out edges. - { - // TODO Fix Gid - spdlog::trace("Recovering outbound edges for vertex {}.", 1); - auto out_size = snapshot.ReadUint(); - if (!out_size) throw RecoveryFailure("Invalid snapshot data!"); - vertex.out_edges.reserve(*out_size); - for (uint64_t j = 0; j < *out_size; ++j) { - auto edge_gid = snapshot.ReadUint(); - if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!"); - last_edge_gid = std::max(last_edge_gid, *edge_gid); - - auto to_gid = snapshot.ReadUint(); - if (!to_gid) throw RecoveryFailure("Invalid snapshot data!"); - auto edge_type = snapshot.ReadUint(); - if (!edge_type) throw RecoveryFailure("Invalid snapshot data!"); - - auto to_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*to_gid).AsInt()}}); - if (to_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid to vertex!"); - - EdgeRef edge_ref(Gid::FromUint(*edge_gid)); - if (items.properties_on_edges) { - if (snapshot_has_edges) { - auto edge = edge_acc.find(Gid::FromUint(*edge_gid)); - if (edge == edge_acc.end()) throw RecoveryFailure("Invalid edge!"); - edge_ref = EdgeRef(&*edge); - } else { - auto [edge, inserted] = edge_acc.insert(Edge{Gid::FromUint(*edge_gid), nullptr}); - edge_ref = EdgeRef(&*edge); - } - } - // TODO Fix Gid - SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid, - name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), 1); - vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), VertexId{kDummyLabelId, {}}, edge_ref); - } - // Increment edge count. We only increment the count here because the - // information is duplicated in in_edges. - *edge_count += *out_size; - } - } - spdlog::info("Connectivity is recovered."); - - // Set initial values for edge/vertex ID generators. - ret.next_edge_id = last_edge_gid + 1; - ret.next_vertex_id = last_vertex_gid + 1; - } - - // Recover indices. - { - spdlog::info("Recovering metadata of indices."); - if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!"); - - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_INDICES) throw RecoveryFailure("Invalid snapshot data!"); - - // Recover label indices. - { - auto size = snapshot.ReadUint(); - if (!size) throw RecoveryFailure("Invalid snapshot data!"); - spdlog::info("Recovering metadata of {} label indices.", *size); - for (uint64_t i = 0; i < *size; ++i) { - auto label = snapshot.ReadUint(); - if (!label) throw RecoveryFailure("Invalid snapshot data!"); - AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label), - "The label index already exists!"); - SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label))); - } - spdlog::info("Metadata of label indices are recovered."); - } - - // Recover label+property indices. - { - auto size = snapshot.ReadUint(); - if (!size) throw RecoveryFailure("Invalid snapshot data!"); - spdlog::info("Recovering metadata of {} label+property indices.", *size); - for (uint64_t i = 0; i < *size; ++i) { - auto label = snapshot.ReadUint(); - if (!label) throw RecoveryFailure("Invalid snapshot data!"); - auto property = snapshot.ReadUint(); - if (!property) throw RecoveryFailure("Invalid snapshot data!"); - AddRecoveredIndexConstraint(&indices_constraints.indices.label_property, - {get_label_from_id(*label), get_property_from_id(*property)}, - "The label+property index already exists!"); - SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})", - name_id_mapper->IdToName(snapshot_id_map.at(*label)), - name_id_mapper->IdToName(snapshot_id_map.at(*property))); - } - spdlog::info("Metadata of label+property indices are recovered."); - } - spdlog::info("Metadata of indices are recovered."); - } - - // Recover constraints. - { - spdlog::info("Recovering metadata of constraints."); - if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!"); - - auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_CONSTRAINTS) throw RecoveryFailure("Invalid snapshot data!"); - - // Recover existence constraints. - { - auto size = snapshot.ReadUint(); - if (!size) throw RecoveryFailure("Invalid snapshot data!"); - spdlog::info("Recovering metadata of {} existence constraints.", *size); - for (uint64_t i = 0; i < *size; ++i) { - auto label = snapshot.ReadUint(); - if (!label) throw RecoveryFailure("Invalid snapshot data!"); - auto property = snapshot.ReadUint(); - if (!property) throw RecoveryFailure("Invalid snapshot data!"); - AddRecoveredIndexConstraint(&indices_constraints.constraints.existence, - {get_label_from_id(*label), get_property_from_id(*property)}, - "The existence constraint already exists!"); - SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})", - name_id_mapper->IdToName(snapshot_id_map.at(*label)), - name_id_mapper->IdToName(snapshot_id_map.at(*property))); - } - spdlog::info("Metadata of existence constraints are recovered."); - } - - // Recover unique constraints. - // Snapshot version should be checked since unique constraints were - // implemented in later versions of snapshot. - if (*version >= kUniqueConstraintVersion) { - auto size = snapshot.ReadUint(); - if (!size) throw RecoveryFailure("Invalid snapshot data!"); - spdlog::info("Recovering metadata of {} unique constraints.", *size); - for (uint64_t i = 0; i < *size; ++i) { - auto label = snapshot.ReadUint(); - if (!label) throw RecoveryFailure("Invalid snapshot data!"); - auto properties_count = snapshot.ReadUint(); - if (!properties_count) throw RecoveryFailure("Invalid snapshot data!"); - std::set properties; - for (uint64_t j = 0; j < *properties_count; ++j) { - auto property = snapshot.ReadUint(); - if (!property) throw RecoveryFailure("Invalid snapshot data!"); - properties.insert(get_property_from_id(*property)); - } - AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties}, - "The unique constraint already exists!"); - SPDLOG_TRACE("Recovered metadata of unique constraints for :{}", - name_id_mapper->IdToName(snapshot_id_map.at(*label))); - } - spdlog::info("Metadata of unique constraints are recovered."); - } - spdlog::info("Metadata of constraints are recovered."); - } - - spdlog::info("Recovering metadata."); - // Recover epoch history - { - if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!"); - - const auto marker = snapshot.ReadMarker(); - if (!marker || *marker != Marker::SECTION_EPOCH_HISTORY) throw RecoveryFailure("Invalid snapshot data!"); - - const auto history_size = snapshot.ReadUint(); - if (!history_size) { - throw RecoveryFailure("Invalid snapshot data!"); - } - - for (int i = 0; i < *history_size; ++i) { - auto maybe_epoch_id = snapshot.ReadString(); - if (!maybe_epoch_id) { - throw RecoveryFailure("Invalid snapshot data!"); - } - const auto maybe_last_commit_timestamp = snapshot.ReadUint(); - if (!maybe_last_commit_timestamp) { - throw RecoveryFailure("Invalid snapshot data!"); - } - epoch_history->emplace_back(std::move(*maybe_epoch_id), *maybe_last_commit_timestamp); - } - } - - spdlog::info("Metadata recovered."); - // Recover timestamp. - ret.next_timestamp = info.start_timestamp + 1; - - // Set success flag (to disable cleanup). - success = true; - - return {info, ret, std::move(indices_constraints)}; -} - -void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count, - VerticesSkipList *vertices, utils::SkipList *edges, NameIdMapper *name_id_mapper, - Indices *indices, Constraints *constraints, Config::Items items, - const VertexValidator &vertex_validator, const std::string &uuid, const std::string_view epoch_id, - const std::deque> &epoch_history, - utils::FileRetainer *file_retainer) { - // Ensure that the storage directory exists. - utils::EnsureDirOrDie(snapshot_directory); - - // Create snapshot file. - auto path = snapshot_directory / MakeSnapshotName(transaction->start_timestamp); - spdlog::info("Starting snapshot creation to {}", path); - Encoder snapshot; - snapshot.Initialize(path, kSnapshotMagic, kVersion); - - // Write placeholder offsets. - uint64_t offset_offsets = 0; - uint64_t offset_edges = 0; - uint64_t offset_vertices = 0; - uint64_t offset_indices = 0; - uint64_t offset_constraints = 0; - uint64_t offset_mapper = 0; - uint64_t offset_metadata = 0; - uint64_t offset_epoch_history = 0; - { - snapshot.WriteMarker(Marker::SECTION_OFFSETS); - offset_offsets = snapshot.GetPosition(); - snapshot.WriteUint(offset_edges); - snapshot.WriteUint(offset_vertices); - snapshot.WriteUint(offset_indices); - snapshot.WriteUint(offset_constraints); - snapshot.WriteUint(offset_mapper); - snapshot.WriteUint(offset_epoch_history); - snapshot.WriteUint(offset_metadata); - } - - // Object counters. - uint64_t edges_count = 0; - uint64_t vertices_count = 0; - - // Mapper data. - std::unordered_set used_ids; - auto write_mapping = [&snapshot, &used_ids](auto mapping) { - used_ids.insert(mapping.AsUint()); - snapshot.WriteUint(mapping.AsUint()); - }; - - // Store all edges. - if (items.properties_on_edges) { - offset_edges = snapshot.GetPosition(); - auto acc = edges->access(); - for (auto &edge : acc) { - // The edge visibility check must be done here manually because we don't - // allow direct access to the edges through the public API. - auto is_visible = !edge.deleted; - auto *delta = edge.delta; - ApplyDeltasForRead(transaction, delta, View::OLD, [&is_visible](const Delta &delta) { - switch (delta.action) { - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - case Delta::Action::SET_PROPERTY: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - break; - case Delta::Action::RECREATE_OBJECT: { - is_visible = true; - break; - } - case Delta::Action::DELETE_OBJECT: { - is_visible = false; - break; - } - } - }); - if (!is_visible) continue; - EdgeRef edge_ref(&edge); - // Here we create an edge accessor that we will use to get the - // properties of the edge. The accessor is created with an invalid - // type and invalid from/to pointers because we don't know them here, - // but that isn't an issue because we won't use that part of the API - // here. - // TODO(jbajic) Fix snapshot with new schema rules - auto ea = EdgeAccessor{edge_ref, - EdgeTypeId::FromUint(0UL), - VertexId{kDummyLabelId, {}}, - VertexId{kDummyLabelId, {}}, - transaction, - indices, - constraints, - items}; - - // Get edge data. - auto maybe_props = ea.Properties(View::OLD); - MG_ASSERT(maybe_props.HasValue(), "Invalid database state!"); - - // Store the edge. - { - snapshot.WriteMarker(Marker::SECTION_EDGE); - snapshot.WriteUint(edge.gid.AsUint()); - const auto &props = maybe_props.GetValue(); - snapshot.WriteUint(props.size()); - for (const auto &item : props) { - write_mapping(item.first); - snapshot.WritePropertyValue(item.second); - } - } - - ++edges_count; - } - } - - // Store all vertices. - { - offset_vertices = snapshot.GetPosition(); - auto acc = vertices->access(); - for (auto &lgo_vertex : acc) { - // The visibility check is implemented for vertices so we use it here. - auto va = VertexAccessor::Create(&lgo_vertex.vertex, transaction, indices, constraints, items, vertex_validator, - View::OLD); - if (!va) continue; - - // Get vertex data. - // TODO (mferencevic): All of these functions could be written into a - // single function so that we traverse the undo deltas only once. - auto maybe_labels = va->Labels(View::OLD); - MG_ASSERT(maybe_labels.HasValue(), "Invalid database state!"); - auto maybe_props = va->Properties(View::OLD); - MG_ASSERT(maybe_props.HasValue(), "Invalid database state!"); - auto maybe_in_edges = va->InEdges(View::OLD); - MG_ASSERT(maybe_in_edges.HasValue(), "Invalid database state!"); - auto maybe_out_edges = va->OutEdges(View::OLD); - MG_ASSERT(maybe_out_edges.HasValue(), "Invalid database state!"); - - // Store the vertex. - { - snapshot.WriteMarker(Marker::SECTION_VERTEX); - // TODO Fix Gid - snapshot.WriteUint(1); - const auto &labels = maybe_labels.GetValue(); - snapshot.WriteUint(labels.size()); - for (const auto &item : labels) { - write_mapping(item); - } - const auto &props = maybe_props.GetValue(); - snapshot.WriteUint(props.size()); - for (const auto &item : props) { - write_mapping(item.first); - snapshot.WritePropertyValue(item.second); - } - const auto &in_edges = maybe_in_edges.GetValue(); - snapshot.WriteUint(in_edges.size()); - // TODO Disabled serialization for vertices - for (const auto &item : in_edges) { - snapshot.WriteUint(item.Gid().AsUint()); - snapshot.WriteUint(1); - write_mapping(item.EdgeType()); - } - const auto &out_edges = maybe_out_edges.GetValue(); - snapshot.WriteUint(out_edges.size()); - for (const auto &item : out_edges) { - snapshot.WriteUint(item.Gid().AsUint()); - snapshot.WriteUint(1); - write_mapping(item.EdgeType()); - } - } - - ++vertices_count; - } - } - - // Write indices. - { - offset_indices = snapshot.GetPosition(); - snapshot.WriteMarker(Marker::SECTION_INDICES); - - // Write label indices. - { - auto label = indices->label_index.ListIndices(); - snapshot.WriteUint(label.size()); - for (const auto &item : label) { - write_mapping(item); - } - } - - // Write label+property indices. - { - auto label_property = indices->label_property_index.ListIndices(); - snapshot.WriteUint(label_property.size()); - for (const auto &item : label_property) { - write_mapping(item.first); - write_mapping(item.second); - } - } - } - - // Write constraints. - { - offset_constraints = snapshot.GetPosition(); - snapshot.WriteMarker(Marker::SECTION_CONSTRAINTS); - - // Write existence constraints. - { - auto existence = ListExistenceConstraints(*constraints); - snapshot.WriteUint(existence.size()); - for (const auto &item : existence) { - write_mapping(item.first); - write_mapping(item.second); - } - } - - // Write unique constraints. - { - auto unique = constraints->unique_constraints.ListConstraints(); - snapshot.WriteUint(unique.size()); - for (const auto &item : unique) { - write_mapping(item.first); - snapshot.WriteUint(item.second.size()); - for (const auto &property : item.second) { - write_mapping(property); - } - } - } - } - - // Write mapper data. - { - offset_mapper = snapshot.GetPosition(); - snapshot.WriteMarker(Marker::SECTION_MAPPER); - snapshot.WriteUint(used_ids.size()); - for (auto item : used_ids) { - snapshot.WriteUint(item); - snapshot.WriteString(name_id_mapper->IdToName(item)); - } - } - - // Write epoch history - { - offset_epoch_history = snapshot.GetPosition(); - snapshot.WriteMarker(Marker::SECTION_EPOCH_HISTORY); - snapshot.WriteUint(epoch_history.size()); - for (const auto &[epoch_id, last_commit_timestamp] : epoch_history) { - snapshot.WriteString(epoch_id); - snapshot.WriteUint(last_commit_timestamp); - } - } - - // Write metadata. - { - offset_metadata = snapshot.GetPosition(); - snapshot.WriteMarker(Marker::SECTION_METADATA); - snapshot.WriteString(uuid); - snapshot.WriteString(epoch_id); - snapshot.WriteUint(transaction->start_timestamp); - snapshot.WriteUint(edges_count); - snapshot.WriteUint(vertices_count); - } - - // Write true offsets. - { - snapshot.SetPosition(offset_offsets); - snapshot.WriteUint(offset_edges); - snapshot.WriteUint(offset_vertices); - snapshot.WriteUint(offset_indices); - snapshot.WriteUint(offset_constraints); - snapshot.WriteUint(offset_mapper); - snapshot.WriteUint(offset_epoch_history); - snapshot.WriteUint(offset_metadata); - } - - // Finalize snapshot file. - snapshot.Finalize(); - spdlog::info("Snapshot creation successful!"); - - // Ensure exactly `snapshot_retention_count` snapshots exist. - std::vector> old_snapshot_files; - { - std::error_code error_code; - for (const auto &item : std::filesystem::directory_iterator(snapshot_directory, error_code)) { - if (!item.is_regular_file()) continue; - if (item.path() == path) continue; - try { - auto info = ReadSnapshotInfo(item.path()); - if (info.uuid != uuid) continue; - old_snapshot_files.emplace_back(info.start_timestamp, item.path()); - } catch (const RecoveryFailure &e) { - spdlog::warn("Found a corrupt snapshot file {} becuase of: {}", item.path(), e.what()); - continue; - } - } - - if (error_code) { - spdlog::error( - utils::MessageWithLink("Couldn't ensure that exactly {} snapshots exist because an error occurred: {}.", - snapshot_retention_count, error_code.message(), "https://memgr.ph/snapshots")); - } - std::sort(old_snapshot_files.begin(), old_snapshot_files.end()); - if (old_snapshot_files.size() > snapshot_retention_count - 1) { - auto num_to_erase = old_snapshot_files.size() - (snapshot_retention_count - 1); - for (size_t i = 0; i < num_to_erase; ++i) { - const auto &[start_timestamp, snapshot_path] = old_snapshot_files[i]; - file_retainer->DeleteFile(snapshot_path); - } - old_snapshot_files.erase( - old_snapshot_files.begin(), - old_snapshot_files.begin() + - static_cast(num_to_erase)); - } - } - - // Ensure that only the absolutely necessary WAL files exist. - if (old_snapshot_files.size() == snapshot_retention_count - 1 && utils::DirExists(wal_directory)) { - std::vector> wal_files; - std::error_code error_code; - for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { - if (!item.is_regular_file()) continue; - try { - auto info = ReadWalInfo(item.path()); - if (info.uuid != uuid) continue; - wal_files.emplace_back(info.seq_num, info.from_timestamp, info.to_timestamp, item.path()); - } catch (const RecoveryFailure &e) { - continue; - } - } - - if (error_code) { - spdlog::error( - utils::MessageWithLink("Couldn't ensure that only the absolutely necessary WAL files exist " - "because an error occurred: {}.", - error_code.message(), "https://memgr.ph/snapshots")); - } - std::sort(wal_files.begin(), wal_files.end()); - uint64_t snapshot_start_timestamp = transaction->start_timestamp; - if (!old_snapshot_files.empty()) { - snapshot_start_timestamp = old_snapshot_files.front().first; - } - std::optional pos = 0; - for (uint64_t i = 0; i < wal_files.size(); ++i) { - const auto &[seq_num, from_timestamp, to_timestamp, wal_path] = wal_files[i]; - if (from_timestamp <= snapshot_start_timestamp) { - pos = i; - } else { - break; - } - } - if (pos && *pos > 0) { - // We need to leave at least one WAL file that contains deltas that were - // created before the oldest snapshot. Because we always leave at least - // one WAL file that contains deltas before the snapshot, this correctly - // handles the edge case when that one file is the current WAL file that - // is being appended to. - for (uint64_t i = 0; i < *pos; ++i) { - const auto &[seq_num, from_timestamp, to_timestamp, wal_path] = wal_files[i]; - file_retainer->DeleteFile(wal_path); - } - } - } -} - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/snapshot.hpp b/src/storage/v3/durability/snapshot.hpp deleted file mode 100644 index 34af826a7..000000000 --- a/src/storage/v3/durability/snapshot.hpp +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -#include "storage/v3/config.hpp" -#include "storage/v3/constraints.hpp" -#include "storage/v3/durability/metadata.hpp" -#include "storage/v3/edge.hpp" -#include "storage/v3/indices.hpp" -#include "storage/v3/name_id_mapper.hpp" -#include "storage/v3/schema_validator.hpp" -#include "storage/v3/transaction.hpp" -#include "storage/v3/vertex.hpp" -#include "utils/file_locker.hpp" -#include "utils/skip_list.hpp" - -namespace memgraph::storage::v3::durability { - -/// Structure used to hold information about a snapshot. -struct SnapshotInfo { - uint64_t offset_edges; - uint64_t offset_vertices; - uint64_t offset_indices; - uint64_t offset_constraints; - uint64_t offset_mapper; - uint64_t offset_epoch_history; - uint64_t offset_metadata; - - std::string uuid; - std::string epoch_id; - uint64_t start_timestamp; - uint64_t edges_count; - uint64_t vertices_count; -}; - -/// Structure used to hold information about the snapshot that has been -/// recovered. -struct RecoveredSnapshot { - SnapshotInfo snapshot_info; - RecoveryInfo recovery_info; - RecoveredIndicesAndConstraints indices_constraints; -}; - -/// Function used to read information about the snapshot file. -/// @throw RecoveryFailure -SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path); - -/// Function used to load the snapshot data into the storage. -/// @throw RecoveryFailure -RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices, - utils::SkipList *edges, - std::deque> *epoch_history, - NameIdMapper *name_id_mapper, uint64_t *edge_count, Config::Items items); - -/// Function used to create a snapshot using the given transaction. -void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory, - const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count, - VerticesSkipList *vertices, utils::SkipList *edges, NameIdMapper *name_id_mapper, - Indices *indices, Constraints *constraints, Config::Items items, - const SchemaValidator &schema_validator, const std::string &uuid, std::string_view epoch_id, - const std::deque> &epoch_history, - utils::FileRetainer *file_retainer); - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/version.hpp b/src/storage/v3/durability/version.hpp deleted file mode 100644 index 9ead68e60..000000000 --- a/src/storage/v3/durability/version.hpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -namespace memgraph::storage::v3::durability { - -// The current version of snapshot and WAL encoding / decoding. -// IMPORTANT: Please bump this version for every snapshot and/or WAL format -// change!!! -const uint64_t kVersion{14}; - -const uint64_t kOldestSupportedVersion{14}; -const uint64_t kUniqueConstraintVersion{13}; - -// Magic values written to the start of a snapshot/WAL file to identify it. -const std::string kSnapshotMagic{"MGsn"}; -const std::string kWalMagic{"MGwl"}; - -static_assert(std::is_same_v); - -// Checks whether the loaded snapshot/WAL version is supported. -inline bool IsVersionSupported(uint64_t version) { return version >= kOldestSupportedVersion && version <= kVersion; } - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/wal.cpp b/src/storage/v3/durability/wal.cpp deleted file mode 100644 index 52959379f..000000000 --- a/src/storage/v3/durability/wal.cpp +++ /dev/null @@ -1,1013 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/durability/wal.hpp" - -#include "storage/v3/delta.hpp" -#include "storage/v3/durability/exceptions.hpp" -#include "storage/v3/durability/paths.hpp" -#include "storage/v3/durability/version.hpp" -#include "storage/v3/edge.hpp" -#include "storage/v3/vertex.hpp" -#include "storage/v3/vertices_skip_list.hpp" -#include "utils/file_locker.hpp" -#include "utils/logging.hpp" - -namespace memgraph::storage::v3::durability { - -// WAL format: -// -// 1) Magic string (non-encoded) -// -// 2) WAL version (non-encoded, little-endian) -// -// 3) Section offsets: -// * offset to the metadata section -// * offset to the first delta in the WAL -// -// 4) Metadata -// * storage UUID -// * sequence number (number indicating the sequence position of this WAL -// file) -// -// 5) Encoded deltas; each delta is written in the following format: -// * commit timestamp -// * action (only one of the actions below are encoded) -// * vertex create, vertex delete -// * gid -// * vertex add label, vertex remove label -// * gid -// * label name -// * vertex set property -// * gid -// * property name -// * property value -// * edge create, edge delete -// * gid -// * edge type name -// * from vertex gid -// * to vertex gid -// * edge set property -// * gid -// * property name -// * property value -// * transaction end (marks that the whole transaction is -// stored in the WAL file) -// * label index create, label index drop -// * label name -// * label property index create, label property index drop, -// existence constraint create, existence constraint drop -// * label name -// * property name -// * unique constraint create, unique constraint drop -// * label name -// * property names -// -// IMPORTANT: When changing WAL encoding/decoding bump the snapshot/WAL version -// in `version.hpp`. - -namespace { - -Marker OperationToMarker(StorageGlobalOperation operation) { - switch (operation) { - case StorageGlobalOperation::LABEL_INDEX_CREATE: - return Marker::DELTA_LABEL_INDEX_CREATE; - case StorageGlobalOperation::LABEL_INDEX_DROP: - return Marker::DELTA_LABEL_INDEX_DROP; - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: - return Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE; - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: - return Marker::DELTA_LABEL_PROPERTY_INDEX_DROP; - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: - return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE; - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: - return Marker::DELTA_EXISTENCE_CONSTRAINT_DROP; - case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: - return Marker::DELTA_UNIQUE_CONSTRAINT_CREATE; - case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: - return Marker::DELTA_UNIQUE_CONSTRAINT_DROP; - } -} - -Marker VertexActionToMarker(Delta::Action action) { - // When converting a Delta to a WAL delta the logic is inverted. That is - // because the Delta's represent undo actions and we want to store redo - // actions. - switch (action) { - case Delta::Action::DELETE_OBJECT: - return Marker::DELTA_VERTEX_CREATE; - case Delta::Action::RECREATE_OBJECT: - return Marker::DELTA_VERTEX_DELETE; - case Delta::Action::SET_PROPERTY: - return Marker::DELTA_VERTEX_SET_PROPERTY; - case Delta::Action::ADD_LABEL: - return Marker::DELTA_VERTEX_REMOVE_LABEL; - case Delta::Action::REMOVE_LABEL: - return Marker::DELTA_VERTEX_ADD_LABEL; - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - return Marker::DELTA_EDGE_DELETE; - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - return Marker::DELTA_EDGE_CREATE; - } -} - -// This function convertes a Marker to a WalDeltaData::Type. It checks for the -// validity of the marker and throws if an invalid marker is specified. -// @throw RecoveryFailure -WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) { - switch (marker) { - case Marker::DELTA_VERTEX_CREATE: - return WalDeltaData::Type::VERTEX_CREATE; - case Marker::DELTA_VERTEX_DELETE: - return WalDeltaData::Type::VERTEX_DELETE; - case Marker::DELTA_VERTEX_ADD_LABEL: - return WalDeltaData::Type::VERTEX_ADD_LABEL; - case Marker::DELTA_VERTEX_REMOVE_LABEL: - return WalDeltaData::Type::VERTEX_REMOVE_LABEL; - case Marker::DELTA_EDGE_CREATE: - return WalDeltaData::Type::EDGE_CREATE; - case Marker::DELTA_EDGE_DELETE: - return WalDeltaData::Type::EDGE_DELETE; - case Marker::DELTA_VERTEX_SET_PROPERTY: - return WalDeltaData::Type::VERTEX_SET_PROPERTY; - case Marker::DELTA_EDGE_SET_PROPERTY: - return WalDeltaData::Type::EDGE_SET_PROPERTY; - case Marker::DELTA_TRANSACTION_END: - return WalDeltaData::Type::TRANSACTION_END; - case Marker::DELTA_LABEL_INDEX_CREATE: - return WalDeltaData::Type::LABEL_INDEX_CREATE; - case Marker::DELTA_LABEL_INDEX_DROP: - return WalDeltaData::Type::LABEL_INDEX_DROP; - case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: - return WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE; - case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: - return WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP; - case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: - return WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE; - case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: - return WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP; - case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE: - return WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE; - case Marker::DELTA_UNIQUE_CONSTRAINT_DROP: - return WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP; - - case Marker::TYPE_NULL: - case Marker::TYPE_BOOL: - case Marker::TYPE_INT: - case Marker::TYPE_DOUBLE: - case Marker::TYPE_STRING: - case Marker::TYPE_LIST: - case Marker::TYPE_MAP: - case Marker::TYPE_TEMPORAL_DATA: - case Marker::TYPE_PROPERTY_VALUE: - case Marker::SECTION_VERTEX: - case Marker::SECTION_EDGE: - case Marker::SECTION_MAPPER: - case Marker::SECTION_METADATA: - case Marker::SECTION_INDICES: - case Marker::SECTION_CONSTRAINTS: - case Marker::SECTION_DELTA: - case Marker::SECTION_EPOCH_HISTORY: - case Marker::SECTION_OFFSETS: - case Marker::VALUE_FALSE: - case Marker::VALUE_TRUE: - throw RecoveryFailure("Invalid WAL data!"); - } -} - -// Function used to either read or skip the current WAL delta data. The WAL -// delta header must be read before calling this function. If the delta data is -// read then the data returned is valid, if the delta data is skipped then the -// returned data is not guaranteed to be set (it could be empty) and shouldn't -// be used. -// @throw RecoveryFailure -template -WalDeltaData ReadSkipWalDeltaData(BaseDecoder *decoder) { - WalDeltaData delta; - - auto action = decoder->ReadMarker(); - if (!action) throw RecoveryFailure("Invalid WAL data!"); - delta.type = MarkerToWalDeltaDataType(*action); - - switch (delta.type) { - case WalDeltaData::Type::VERTEX_CREATE: - case WalDeltaData::Type::VERTEX_DELETE: { - auto gid = decoder->ReadUint(); - if (!gid) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_create_delete.gid = Gid::FromUint(*gid); - break; - } - case WalDeltaData::Type::VERTEX_ADD_LABEL: - case WalDeltaData::Type::VERTEX_REMOVE_LABEL: { - auto gid = decoder->ReadUint(); - if (!gid) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_add_remove_label.gid = Gid::FromUint(*gid); - if constexpr (read_data) { - auto label = decoder->ReadString(); - if (!label) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_add_remove_label.label = std::move(*label); - } else { - if (!decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - } - break; - } - case WalDeltaData::Type::VERTEX_SET_PROPERTY: - case WalDeltaData::Type::EDGE_SET_PROPERTY: { - auto gid = decoder->ReadUint(); - if (!gid) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_edge_set_property.gid = Gid::FromUint(*gid); - if constexpr (read_data) { - auto property = decoder->ReadString(); - if (!property) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_edge_set_property.property = std::move(*property); - auto value = decoder->ReadPropertyValue(); - if (!value) throw RecoveryFailure("Invalid WAL data!"); - delta.vertex_edge_set_property.value = std::move(*value); - } else { - if (!decoder->SkipString() || !decoder->SkipPropertyValue()) throw RecoveryFailure("Invalid WAL data!"); - } - break; - } - case WalDeltaData::Type::EDGE_CREATE: - case WalDeltaData::Type::EDGE_DELETE: { - auto gid = decoder->ReadUint(); - if (!gid) throw RecoveryFailure("Invalid WAL data!"); - delta.edge_create_delete.gid = Gid::FromUint(*gid); - if constexpr (read_data) { - auto edge_type = decoder->ReadString(); - if (!edge_type) throw RecoveryFailure("Invalid WAL data!"); - delta.edge_create_delete.edge_type = std::move(*edge_type); - } else { - if (!decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - } - auto from_gid = decoder->ReadUint(); - if (!from_gid) throw RecoveryFailure("Invalid WAL data!"); - delta.edge_create_delete.from_vertex = Gid::FromUint(*from_gid); - auto to_gid = decoder->ReadUint(); - if (!to_gid) throw RecoveryFailure("Invalid WAL data!"); - delta.edge_create_delete.to_vertex = Gid::FromUint(*to_gid); - break; - } - case WalDeltaData::Type::TRANSACTION_END: - break; - case WalDeltaData::Type::LABEL_INDEX_CREATE: - case WalDeltaData::Type::LABEL_INDEX_DROP: { - if constexpr (read_data) { - auto label = decoder->ReadString(); - if (!label) throw RecoveryFailure("Invalid WAL data!"); - delta.operation_label.label = std::move(*label); - } else { - if (!decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - } - break; - } - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { - if constexpr (read_data) { - auto label = decoder->ReadString(); - if (!label) throw RecoveryFailure("Invalid WAL data!"); - delta.operation_label_property.label = std::move(*label); - auto property = decoder->ReadString(); - if (!property) throw RecoveryFailure("Invalid WAL data!"); - delta.operation_label_property.property = std::move(*property); - } else { - if (!decoder->SkipString() || !decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - } - break; - } - case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: - case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { - if constexpr (read_data) { - auto label = decoder->ReadString(); - if (!label) throw RecoveryFailure("Invalid WAL data!"); - delta.operation_label_properties.label = std::move(*label); - auto properties_count = decoder->ReadUint(); - if (!properties_count) throw RecoveryFailure("Invalid WAL data!"); - for (uint64_t i = 0; i < *properties_count; ++i) { - auto property = decoder->ReadString(); - if (!property) throw RecoveryFailure("Invalid WAL data!"); - delta.operation_label_properties.properties.emplace(std::move(*property)); - } - } else { - if (!decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - auto properties_count = decoder->ReadUint(); - if (!properties_count) throw RecoveryFailure("Invalid WAL data!"); - for (uint64_t i = 0; i < *properties_count; ++i) { - if (!decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); - } - } - } - } - - return delta; -} - -} // namespace - -// Function used to read information about the WAL file. -WalInfo ReadWalInfo(const std::filesystem::path &path) { - // Check magic and version. - Decoder wal; - auto version = wal.Initialize(path, kWalMagic); - if (!version) throw RecoveryFailure("Couldn't read WAL magic and/or version!"); - if (!IsVersionSupported(*version)) throw RecoveryFailure("Invalid WAL version!"); - - // Prepare return value. - WalInfo info; - - // Read offsets. - { - auto marker = wal.ReadMarker(); - if (!marker || *marker != Marker::SECTION_OFFSETS) throw RecoveryFailure("Invalid WAL data!"); - - auto wal_size = wal.GetSize(); - if (!wal_size) throw RecoveryFailure("Invalid WAL data!"); - - auto read_offset = [&wal, wal_size] { - auto maybe_offset = wal.ReadUint(); - if (!maybe_offset) throw RecoveryFailure("Invalid WAL format!"); - auto offset = *maybe_offset; - if (offset > *wal_size) throw RecoveryFailure("Invalid WAL format!"); - return offset; - }; - - info.offset_metadata = read_offset(); - info.offset_deltas = read_offset(); - } - - // Read metadata. - { - wal.SetPosition(info.offset_metadata); - - auto marker = wal.ReadMarker(); - if (!marker || *marker != Marker::SECTION_METADATA) throw RecoveryFailure("Invalid WAL data!"); - - auto maybe_uuid = wal.ReadString(); - if (!maybe_uuid) throw RecoveryFailure("Invalid WAL data!"); - info.uuid = std::move(*maybe_uuid); - - auto maybe_epoch_id = wal.ReadString(); - if (!maybe_epoch_id) throw RecoveryFailure("Invalid WAL data!"); - info.epoch_id = std::move(*maybe_epoch_id); - - auto maybe_seq_num = wal.ReadUint(); - if (!maybe_seq_num) throw RecoveryFailure("Invalid WAL data!"); - info.seq_num = *maybe_seq_num; - } - - // Read deltas. - info.num_deltas = 0; - auto validate_delta = [&wal]() -> std::optional> { - try { - auto timestamp = ReadWalDeltaHeader(&wal); - auto type = SkipWalDeltaData(&wal); - return {{timestamp, IsWalDeltaDataTypeTransactionEnd(type)}}; - } catch (const RecoveryFailure &) { - return std::nullopt; - } - }; - auto size = wal.GetSize(); - // Here we read the whole file and determine the number of valid deltas. A - // delta is valid only if all of its data can be successfully read. This - // allows us to recover data from WAL files that are corrupt at the end (eg. - // because of power loss) but are still valid at the beginning. While reading - // the deltas we only count deltas which are a part of a fully valid - // transaction (indicated by a TRANSACTION_END delta or any other - // non-transactional operation). - std::optional current_timestamp; - uint64_t num_deltas = 0; - while (wal.GetPosition() != size) { - auto ret = validate_delta(); - if (!ret) break; - auto [timestamp, is_end_of_transaction] = *ret; - if (!current_timestamp) current_timestamp = timestamp; - if (*current_timestamp != timestamp) break; - ++num_deltas; - if (is_end_of_transaction) { - if (info.num_deltas == 0) { - info.from_timestamp = timestamp; - info.to_timestamp = timestamp; - } - if (timestamp < info.from_timestamp || timestamp < info.to_timestamp) break; - info.to_timestamp = timestamp; - info.num_deltas += num_deltas; - current_timestamp = std::nullopt; - num_deltas = 0; - } - } - - if (info.num_deltas == 0) throw RecoveryFailure("Invalid WAL data!"); - - return info; -} - -bool operator==(const WalDeltaData &a, const WalDeltaData &b) { - if (a.type != b.type) return false; - switch (a.type) { - case WalDeltaData::Type::VERTEX_CREATE: - case WalDeltaData::Type::VERTEX_DELETE: - return a.vertex_create_delete.gid == b.vertex_create_delete.gid; - - case WalDeltaData::Type::VERTEX_ADD_LABEL: - case WalDeltaData::Type::VERTEX_REMOVE_LABEL: - return a.vertex_add_remove_label.gid == b.vertex_add_remove_label.gid && - a.vertex_add_remove_label.label == b.vertex_add_remove_label.label; - - case WalDeltaData::Type::VERTEX_SET_PROPERTY: - case WalDeltaData::Type::EDGE_SET_PROPERTY: - return a.vertex_edge_set_property.gid == b.vertex_edge_set_property.gid && - a.vertex_edge_set_property.property == b.vertex_edge_set_property.property && - a.vertex_edge_set_property.value == b.vertex_edge_set_property.value; - - case WalDeltaData::Type::EDGE_CREATE: - case WalDeltaData::Type::EDGE_DELETE: - return a.edge_create_delete.gid == b.edge_create_delete.gid && - a.edge_create_delete.edge_type == b.edge_create_delete.edge_type && - a.edge_create_delete.from_vertex == b.edge_create_delete.from_vertex && - a.edge_create_delete.to_vertex == b.edge_create_delete.to_vertex; - - case WalDeltaData::Type::TRANSACTION_END: - return true; - - case WalDeltaData::Type::LABEL_INDEX_CREATE: - case WalDeltaData::Type::LABEL_INDEX_DROP: - return a.operation_label.label == b.operation_label.label; - - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: - return a.operation_label_property.label == b.operation_label_property.label && - a.operation_label_property.property == b.operation_label_property.property; - case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: - case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: - return a.operation_label_properties.label == b.operation_label_properties.label && - a.operation_label_properties.properties == b.operation_label_properties.properties; - } -} -bool operator!=(const WalDeltaData &a, const WalDeltaData &b) { return !(a == b); } - -// Function used to read the WAL delta header. The function returns the delta -// timestamp. -uint64_t ReadWalDeltaHeader(BaseDecoder *decoder) { - auto marker = decoder->ReadMarker(); - if (!marker || *marker != Marker::SECTION_DELTA) throw RecoveryFailure("Invalid WAL data!"); - - auto timestamp = decoder->ReadUint(); - if (!timestamp) throw RecoveryFailure("Invalid WAL data!"); - return *timestamp; -} - -// Function used to read the current WAL delta data. The WAL delta header must -// be read before calling this function. -WalDeltaData ReadWalDeltaData(BaseDecoder *decoder) { return ReadSkipWalDeltaData(decoder); } - -// Function used to skip the current WAL delta data. The WAL delta header must -// be read before calling this function. -WalDeltaData::Type SkipWalDeltaData(BaseDecoder *decoder) { - auto delta = ReadSkipWalDeltaData(decoder); - return delta.type; -} - -void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Items items, const Delta &delta, - const Vertex &vertex, uint64_t timestamp) { - // When converting a Delta to a WAL delta the logic is inverted. That is - // because the Delta's represent undo actions and we want to store redo - // actions. - // encoder->WriteMarker(Marker::SECTION_DELTA); - // encoder->WriteUint(timestamp); - // switch (delta.action) { - // case Delta::Action::DELETE_OBJECT: - // case Delta::Action::RECREATE_OBJECT: { - // encoder->WriteMarker(VertexActionToMarker(delta.action)); - // encoder->WriteUint(vertex.Gid().AsUint()); - // break; - // } - // case Delta::Action::SET_PROPERTY: { - // encoder->WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY); - // encoder->WriteUint(vertex.Gid().AsUint()); - // encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint())); - // // The property value is the value that is currently stored in the - // // vertex. - // // TODO (mferencevic): Mitigate the memory allocation introduced here - // // (with the `GetProperty` call). It is the only memory allocation in the - // // entire WAL file writing logic. - // encoder->WritePropertyValue(vertex.properties.GetProperty(delta.property.key)); - // break; - // } - // case Delta::Action::ADD_LABEL: - // case Delta::Action::REMOVE_LABEL: { - // encoder->WriteMarker(VertexActionToMarker(delta.action)); - // encoder->WriteUint(vertex.Gid().AsUint()); - // encoder->WriteString(name_id_mapper->IdToName(delta.label.AsUint())); - // break; - // } - // case Delta::Action::ADD_OUT_EDGE: - // case Delta::Action::REMOVE_OUT_EDGE: { - // encoder->WriteMarker(VertexActionToMarker(delta.action)); - // if (items.properties_on_edges) { - // encoder->WriteUint(delta.vertex_edge.edge.ptr->gid.AsUint()); - // } else { - // encoder->WriteUint(delta.vertex_edge.edge.gid.AsUint()); - // } - // encoder->WriteString(name_id_mapper->IdToName(delta.vertex_edge.edge_type.AsUint())); - // encoder->WriteUint(vertex.Gid().AsUint()); - // encoder->WriteUint(delta.vertex_edge.vertex->Gid().AsUint()); - // break; - // } - // case Delta::Action::ADD_IN_EDGE: - // case Delta::Action::REMOVE_IN_EDGE: - // // These actions are already encoded in the *_OUT_EDGE actions. This - // // function should never be called for this type of deltas. - // LOG_FATAL("Invalid delta action!"); - // } -} - -void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta &delta, const Edge &edge, - uint64_t timestamp) { - // When converting a Delta to a WAL delta the logic is inverted. That is - // because the Delta's represent undo actions and we want to store redo - // actions. - // encoder->WriteMarker(Marker::SECTION_DELTA); - // encoder->WriteUint(timestamp); - // switch (delta.action) { - // case Delta::Action::SET_PROPERTY: { - // encoder->WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY); - // encoder->WriteUint(edge.gid.AsUint()); - // encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint())); - // // The property value is the value that is currently stored in the - // // edge. - // // TODO (mferencevic): Mitigate the memory allocation introduced here - // // (with the `GetProperty` call). It is the only memory allocation in the - // // entire WAL file writing logic. - // encoder->WritePropertyValue(edge.properties.GetProperty(delta.property.key)); - // break; - // } - // case Delta::Action::DELETE_OBJECT: - // case Delta::Action::RECREATE_OBJECT: - // // These actions are already encoded in vertex *_OUT_EDGE actions. Also, - // // these deltas don't contain any information about the from vertex, to - // // vertex or edge type so they are useless. This function should never - // // be called for this type of deltas. - // LOG_FATAL("Invalid delta action!"); - // case Delta::Action::ADD_LABEL: - // case Delta::Action::REMOVE_LABEL: - // case Delta::Action::ADD_OUT_EDGE: - // case Delta::Action::REMOVE_OUT_EDGE: - // case Delta::Action::ADD_IN_EDGE: - // case Delta::Action::REMOVE_IN_EDGE: - // // These deltas shouldn't appear for edges. - // LOG_FATAL("Invalid database state!"); - // } -} - -void EncodeTransactionEnd(BaseEncoder *encoder, uint64_t timestamp) { - encoder->WriteMarker(Marker::SECTION_DELTA); - encoder->WriteUint(timestamp); - encoder->WriteMarker(Marker::DELTA_TRANSACTION_END); -} - -void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageGlobalOperation operation, - LabelId label, const std::set &properties, uint64_t timestamp) { - encoder->WriteMarker(Marker::SECTION_DELTA); - encoder->WriteUint(timestamp); - switch (operation) { - case StorageGlobalOperation::LABEL_INDEX_CREATE: - case StorageGlobalOperation::LABEL_INDEX_DROP: { - MG_ASSERT(properties.empty(), "Invalid function call!"); - encoder->WriteMarker(OperationToMarker(operation)); - encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); - break; - } - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: { - MG_ASSERT(properties.size() == 1, "Invalid function call!"); - encoder->WriteMarker(OperationToMarker(operation)); - encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); - encoder->WriteString(name_id_mapper->IdToName((*properties.begin()).AsUint())); - break; - } - case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: - case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: { - MG_ASSERT(!properties.empty(), "Invalid function call!"); - encoder->WriteMarker(OperationToMarker(operation)); - encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); - encoder->WriteUint(properties.size()); - for (const auto &property : properties) { - encoder->WriteString(name_id_mapper->IdToName(property.AsUint())); - } - break; - } - } -} - -RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints, - const std::optional last_loaded_timestamp, VerticesSkipList *vertices, - utils::SkipList *edges, NameIdMapper * /*name_id_mapper*/, uint64_t * /*edge_count*/, - Config::Items items) { - spdlog::info("Trying to load WAL file {}.", path); - RecoveryInfo ret; - - Decoder wal; - auto version = wal.Initialize(path, kWalMagic); - if (!version) throw RecoveryFailure("Couldn't read WAL magic and/or version!"); - if (!IsVersionSupported(*version)) throw RecoveryFailure("Invalid WAL version!"); - - // Read wal info. - auto info = ReadWalInfo(path); - ret.last_commit_timestamp = info.to_timestamp; - - // Check timestamp. - if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) { - spdlog::info("Skip loading WAL file because it is too old."); - return ret; - } - - // Recover deltas. - wal.SetPosition(info.offset_deltas); - uint64_t deltas_applied = 0; - auto edge_acc = edges->access(); - auto vertex_acc = vertices->access(); - spdlog::info("WAL file contains {} deltas.", info.num_deltas); - // for (uint64_t i = 0; i < info.num_deltas; ++i) { - // // Read WAL delta header to find out the delta timestamp. - // auto timestamp = ReadWalDeltaHeader(&wal); - - // if (!last_loaded_timestamp || timestamp > *last_loaded_timestamp) { - // // This delta should be loaded. - // auto delta = ReadWalDeltaData(&wal); - // switch (delta.type) { - // case WalDeltaData::Type::VERTEX_CREATE: { - // auto [vertex, inserted] = vertex_acc.insert({Vertex{delta.vertex_create_delete.gid, nullptr}}); - // if (!inserted) throw RecoveryFailure("The vertex must be inserted here!"); - - // ret.next_vertex_id = std::max(ret.next_vertex_id, delta.vertex_create_delete.gid.AsUint() + 1); - - // break; - // } - // case WalDeltaData::Type::VERTEX_DELETE: { - // auto lgo_vertex_it = vertex_acc.find(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}}); - // if (lgo_vertex_it == vertex_acc.end()) { - // throw RecoveryFailure("The vertex doesn't exist!"); - // } - // auto &vertex = lgo_vertex_it->vertex; - // if (!vertex.in_edges.empty() || !vertex.out_edges.empty()) - // throw RecoveryFailure("The vertex can't be deleted because it still has edges!"); - - // if (!vertex_acc.remove(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}})) - // throw RecoveryFailure("The vertex must be removed here!"); - - // break; - // } - // case WalDeltaData::Type::VERTEX_ADD_LABEL: - // case WalDeltaData::Type::VERTEX_REMOVE_LABEL: { - // auto lgo_vertex_it = - // vertex_acc.find(std::vector{PropertyValue{delta.vertex_add_remove_label.gid.AsInt()}}); if (lgo_vertex_it - // == vertex_acc.end()) { - // throw RecoveryFailure("The vertex doesn't exist!"); - // } - // auto &vertex = lgo_vertex_it->vertex; - - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.vertex_add_remove_label.label)); - // auto it = std::find(vertex.labels.begin(), vertex.labels.end(), label_id); - - // if (delta.type == WalDeltaData::Type::VERTEX_ADD_LABEL) { - // if (it != vertex.labels.end()) throw RecoveryFailure("The vertex already has the label!"); - // vertex.labels.push_back(label_id); - // } else { - // if (it == vertex.labels.end()) throw RecoveryFailure("The vertex doesn't have the label!"); - // std::swap(*it, vertex.labels.back()); - // vertex.labels.pop_back(); - // } - - // break; - // } - // case WalDeltaData::Type::VERTEX_SET_PROPERTY: { - // auto lgo_vertex_it = - // vertex_acc.find(std::vector{PropertyValue{delta.vertex_edge_set_property.gid.AsInt()}}); if (lgo_vertex_it - // == vertex_acc.end()) { - // throw RecoveryFailure("The vertex doesn't exist!"); - // } - - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property)); - // auto &property_value = delta.vertex_edge_set_property.value; - - // lgo_vertex_it->vertex.properties.SetProperty(property_id, property_value); - - // break; - // } - // case WalDeltaData::Type::EDGE_CREATE: { - // auto from_lgo_vertex = - // vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}}); - // if (from_lgo_vertex == vertex_acc.end()) { - // throw RecoveryFailure("The from vertex doesn't exist!"); - // } - // auto to_lgo_vertex = - // vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}}); if (to_lgo_vertex - // == vertex_acc.end()) { - // throw RecoveryFailure("The to vertex doesn't exist!"); - // } - // auto &from_vertex = from_lgo_vertex->vertex; - // auto &to_vertex = to_lgo_vertex->vertex; - - // auto edge_gid = delta.edge_create_delete.gid; - // auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type)); - // EdgeRef edge_ref(edge_gid); - // if (items.properties_on_edges) { - // auto [edge, inserted] = edge_acc.insert(Edge{edge_gid, nullptr}); - // if (!inserted) throw RecoveryFailure("The edge must be inserted here!"); - // edge_ref = EdgeRef(&*edge); - // } - // { - // std::tuple link{edge_type_id, &to_vertex, edge_ref}; - // auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link); - // if (it != from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex already has this edge!"); - // from_vertex.out_edges.push_back(link); - // } - // { - // std::tuple link{edge_type_id, &from_vertex, edge_ref}; - // auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link); - // if (it != to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex already has this edge!"); - // to_vertex.in_edges.push_back(link); - // } - - // ret.next_edge_id = std::max(ret.next_edge_id, edge_gid.AsUint() + 1); - - // // Increment edge count. - // *edge_count += 1; - - // break; - // } - // case WalDeltaData::Type::EDGE_DELETE: { - // auto from_lgo_vertex = - // vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}}); - // if (from_lgo_vertex == vertex_acc.end()) { - // throw RecoveryFailure("The from vertex doesn't exist!"); - // } - // auto to_lgo_vertex = - // vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}}); if (to_lgo_vertex - // == vertex_acc.end()) { - // throw RecoveryFailure("The to vertex doesn't exist!"); - // } - // auto &from_vertex = from_lgo_vertex->vertex; - // auto &to_vertex = to_lgo_vertex->vertex; - - // auto edge_gid = delta.edge_create_delete.gid; - // auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type)); - // EdgeRef edge_ref(edge_gid); - // if (items.properties_on_edges) { - // auto edge = edge_acc.find(edge_gid); - // if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!"); - // edge_ref = EdgeRef(&*edge); - // } - // { - // std::tuple link{edge_type_id, &to_vertex, edge_ref}; - // auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link); - // if (it == from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex doesn't have this edge!"); - // std::swap(*it, from_vertex.out_edges.back()); - // from_vertex.out_edges.pop_back(); - // } - // { - // std::tuple link{edge_type_id, &from_vertex, edge_ref}; - // auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link); - // if (it == to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex doesn't have this edge!"); - // std::swap(*it, to_vertex.in_edges.back()); - // to_vertex.in_edges.pop_back(); - // } - // if (items.properties_on_edges) { - // if (!edge_acc.remove(edge_gid)) throw RecoveryFailure("The edge must be removed here!"); - // } - - // // Decrement edge count. - // *edge_count += -1; - - // break; - // } - // case WalDeltaData::Type::EDGE_SET_PROPERTY: { - // if (!items.properties_on_edges) - // throw RecoveryFailure( - // "The WAL has properties on edges, but the storage is " - // "configured without properties on edges!"); - // auto edge = edge_acc.find(delta.vertex_edge_set_property.gid); - // if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!"); - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property)); - // auto &property_value = delta.vertex_edge_set_property.value; - // edge->properties.SetProperty(property_id, property_value); - // break; - // } - // case WalDeltaData::Type::TRANSACTION_END: - // break; - // case WalDeltaData::Type::LABEL_INDEX_CREATE: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label)); - // AddRecoveredIndexConstraint(&indices_constraints->indices.label, label_id, "The label index already - // exists!"); break; - // } - // case WalDeltaData::Type::LABEL_INDEX_DROP: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label)); - // RemoveRecoveredIndexConstraint(&indices_constraints->indices.label, label_id, - // "The label index doesn't exist!"); - // break; - // } - // case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); - // AddRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id}, - // "The label property index already exists!"); - // break; - // } - // case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); - // RemoveRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id}, - // "The label property index doesn't exist!"); - // break; - // } - // case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); - // AddRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id}, - // "The existence constraint already exists!"); - // break; - // } - // case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); - // auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); - // RemoveRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id}, - // "The existence constraint doesn't exist!"); - // break; - // } - // case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label)); - // std::set property_ids; - // for (const auto &prop : delta.operation_label_properties.properties) { - // property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop))); - // } - // AddRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids}, - // "The unique constraint already exists!"); - // break; - // } - // case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { - // auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label)); - // std::set property_ids; - // for (const auto &prop : delta.operation_label_properties.properties) { - // property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop))); - // } - // RemoveRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids}, - // "The unique constraint doesn't exist!"); - // break; - // } - // } - // ret.next_timestamp = std::max(ret.next_timestamp, timestamp + 1); - // ++deltas_applied; - // } else { - // // This delta should be skipped. - // SkipWalDeltaData(&wal); - // } - // } - - // spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied, - // info.num_deltas - deltas_applied); - - return ret; -} - -WalFile::WalFile(const std::filesystem::path &wal_directory, const std::string_view uuid, - const std::string_view epoch_id, Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num, - utils::FileRetainer *file_retainer) - : items_(items), - name_id_mapper_(name_id_mapper), - path_(wal_directory / MakeWalName()), - from_timestamp_(0), - to_timestamp_(0), - count_(0), - seq_num_(seq_num), - file_retainer_(file_retainer) { - // Ensure that the storage directory exists. - utils::EnsureDirOrDie(wal_directory); - - // Initialize the WAL file. - wal_.Initialize(path_, kWalMagic, kVersion); - - // Write placeholder offsets. - uint64_t offset_offsets = 0; - uint64_t offset_metadata = 0; - uint64_t offset_deltas = 0; - wal_.WriteMarker(Marker::SECTION_OFFSETS); - offset_offsets = wal_.GetPosition(); - wal_.WriteUint(offset_metadata); - wal_.WriteUint(offset_deltas); - - // Write metadata. - offset_metadata = wal_.GetPosition(); - wal_.WriteMarker(Marker::SECTION_METADATA); - wal_.WriteString(uuid); - wal_.WriteString(epoch_id); - wal_.WriteUint(seq_num); - - // Write final offsets. - offset_deltas = wal_.GetPosition(); - wal_.SetPosition(offset_offsets); - wal_.WriteUint(offset_metadata); - wal_.WriteUint(offset_deltas); - wal_.SetPosition(offset_deltas); - - // Sync the initial data. - wal_.Sync(); -} - -WalFile::WalFile(std::filesystem::path current_wal_path, Config::Items items, NameIdMapper *name_id_mapper, - uint64_t seq_num, uint64_t from_timestamp, uint64_t to_timestamp, uint64_t count, - utils::FileRetainer *file_retainer) - : items_(items), - name_id_mapper_(name_id_mapper), - path_(std::move(current_wal_path)), - from_timestamp_(from_timestamp), - to_timestamp_(to_timestamp), - count_(count), - seq_num_(seq_num), - file_retainer_(file_retainer) { - wal_.OpenExisting(path_); -} - -void WalFile::FinalizeWal() { - if (count_ != 0) { - wal_.Finalize(); - // Rename file. - std::filesystem::path new_path(path_); - new_path.replace_filename(RemakeWalName(path_.filename(), from_timestamp_, to_timestamp_)); - - utils::CopyFile(path_, new_path); - wal_.Close(); - file_retainer_->DeleteFile(path_); - path_ = std::move(new_path); - } -} - -void WalFile::DeleteWal() { - wal_.Close(); - file_retainer_->DeleteFile(path_); -} - -WalFile::~WalFile() { - if (count_ == 0) { - // Remove empty WAL file. - utils::DeleteFile(path_); - } -} - -void WalFile::AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp) { - EncodeDelta(&wal_, name_id_mapper_, items_, delta, vertex, timestamp); - UpdateStats(timestamp); -} - -void WalFile::AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp) { - EncodeDelta(&wal_, name_id_mapper_, delta, edge, timestamp); - UpdateStats(timestamp); -} - -void WalFile::AppendTransactionEnd(uint64_t timestamp) { - EncodeTransactionEnd(&wal_, timestamp); - UpdateStats(timestamp); -} - -void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label, const std::set &properties, - uint64_t timestamp) { - EncodeOperation(&wal_, name_id_mapper_, operation, label, properties, timestamp); - UpdateStats(timestamp); -} - -void WalFile::Sync() { wal_.Sync(); } - -uint64_t WalFile::GetSize() { return wal_.GetSize(); } - -uint64_t WalFile::SequenceNumber() const { return seq_num_; } - -void WalFile::UpdateStats(uint64_t timestamp) { - if (count_ == 0) from_timestamp_ = timestamp; - to_timestamp_ = timestamp; - count_ += 1; -} - -void WalFile::DisableFlushing() { wal_.DisableFlushing(); } - -void WalFile::EnableFlushing() { wal_.EnableFlushing(); } - -void WalFile::TryFlushing() { wal_.TryFlushing(); } - -std::pair WalFile::CurrentFileBuffer() const { return wal_.CurrentFileBuffer(); } - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/durability/wal.hpp b/src/storage/v3/durability/wal.hpp deleted file mode 100644 index a59789edb..000000000 --- a/src/storage/v3/durability/wal.hpp +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include - -#include "storage/v3/config.hpp" -#include "storage/v3/delta.hpp" -#include "storage/v3/durability/metadata.hpp" -#include "storage/v3/durability/serialization.hpp" -#include "storage/v3/edge.hpp" -#include "storage/v3/id_types.hpp" -#include "storage/v3/name_id_mapper.hpp" -#include "storage/v3/property_value.hpp" -#include "storage/v3/vertex.hpp" -#include "storage/v3/vertices_skip_list.hpp" -#include "utils/file_locker.hpp" -#include "utils/skip_list.hpp" - -namespace memgraph::storage::v3::durability { - -/// Structure used to hold information about a WAL. -struct WalInfo { - uint64_t offset_metadata; - uint64_t offset_deltas; - - std::string uuid; - std::string epoch_id; - uint64_t seq_num; - uint64_t from_timestamp; - uint64_t to_timestamp; - uint64_t num_deltas; -}; - -/// Structure used to return loaded WAL delta data. -struct WalDeltaData { - enum class Type { - VERTEX_CREATE, - VERTEX_DELETE, - VERTEX_ADD_LABEL, - VERTEX_REMOVE_LABEL, - VERTEX_SET_PROPERTY, - EDGE_CREATE, - EDGE_DELETE, - EDGE_SET_PROPERTY, - TRANSACTION_END, - LABEL_INDEX_CREATE, - LABEL_INDEX_DROP, - LABEL_PROPERTY_INDEX_CREATE, - LABEL_PROPERTY_INDEX_DROP, - EXISTENCE_CONSTRAINT_CREATE, - EXISTENCE_CONSTRAINT_DROP, - UNIQUE_CONSTRAINT_CREATE, - UNIQUE_CONSTRAINT_DROP, - }; - - Type type{Type::TRANSACTION_END}; - - struct { - Gid gid; - } vertex_create_delete; - - struct { - Gid gid; - std::string label; - } vertex_add_remove_label; - - struct { - Gid gid; - std::string property; - PropertyValue value; - } vertex_edge_set_property; - - struct { - Gid gid; - std::string edge_type; - Gid from_vertex; - Gid to_vertex; - } edge_create_delete; - - struct { - std::string label; - } operation_label; - - struct { - std::string label; - std::string property; - } operation_label_property; - - struct { - std::string label; - std::set properties; - } operation_label_properties; -}; - -bool operator==(const WalDeltaData &a, const WalDeltaData &b); -bool operator!=(const WalDeltaData &a, const WalDeltaData &b); - -/// Enum used to indicate a global database operation that isn't transactional. -enum class StorageGlobalOperation { - LABEL_INDEX_CREATE, - LABEL_INDEX_DROP, - LABEL_PROPERTY_INDEX_CREATE, - LABEL_PROPERTY_INDEX_DROP, - EXISTENCE_CONSTRAINT_CREATE, - EXISTENCE_CONSTRAINT_DROP, - UNIQUE_CONSTRAINT_CREATE, - UNIQUE_CONSTRAINT_DROP, -}; - -constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) { - switch (type) { - // These delta actions are all found inside transactions so they don't - // indicate a transaction end. - case WalDeltaData::Type::VERTEX_CREATE: - case WalDeltaData::Type::VERTEX_DELETE: - case WalDeltaData::Type::VERTEX_ADD_LABEL: - case WalDeltaData::Type::VERTEX_REMOVE_LABEL: - case WalDeltaData::Type::EDGE_CREATE: - case WalDeltaData::Type::EDGE_DELETE: - case WalDeltaData::Type::VERTEX_SET_PROPERTY: - case WalDeltaData::Type::EDGE_SET_PROPERTY: - return false; - - // This delta explicitly indicates that a transaction is done. - case WalDeltaData::Type::TRANSACTION_END: - // These operations aren't transactional and they are encoded only using - // a single delta, so they each individually mark the end of their - // 'transaction'. - case WalDeltaData::Type::LABEL_INDEX_CREATE: - case WalDeltaData::Type::LABEL_INDEX_DROP: - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: - case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: - case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: - case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: - case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: - return true; - } -} - -/// Function used to read information about the WAL file. -/// @throw RecoveryFailure -WalInfo ReadWalInfo(const std::filesystem::path &path); - -/// Function used to read the WAL delta header. The function returns the delta -/// timestamp. -/// @throw RecoveryFailure -uint64_t ReadWalDeltaHeader(BaseDecoder *decoder); - -/// Function used to read the current WAL delta data. The function returns the -/// read delta data. The WAL delta header must be read before calling this -/// function. -/// @throw RecoveryFailure -WalDeltaData ReadWalDeltaData(BaseDecoder *decoder); - -/// Function used to skip the current WAL delta data. The function returns the -/// skipped delta type. The WAL delta header must be read before calling this -/// function. -/// @throw RecoveryFailure -WalDeltaData::Type SkipWalDeltaData(BaseDecoder *decoder); - -/// Function used to encode a `Delta` that originated from a `Vertex`. -void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Items items, const Delta &delta, - const Vertex &vertex, uint64_t timestamp); - -/// Function used to encode a `Delta` that originated from an `Edge`. -void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta &delta, const Edge &edge, - uint64_t timestamp); - -/// Function used to encode the transaction end. -void EncodeTransactionEnd(BaseEncoder *encoder, uint64_t timestamp); - -/// Function used to encode non-transactional operation. -void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageGlobalOperation operation, - LabelId label, const std::set &properties, uint64_t timestamp); - -/// Function used to load the WAL data into the storage. -/// @throw RecoveryFailure -RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints, - std::optional last_loaded_timestamp, VerticesSkipList *vertices, - utils::SkipList *edges, NameIdMapper *name_id_mapper, uint64_t *edge_count, - Config::Items items); - -/// WalFile class used to append deltas and operations to the WAL file. -class WalFile { - public: - WalFile(const std::filesystem::path &wal_directory, std::string_view uuid, std::string_view epoch_id, - Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num, utils::FileRetainer *file_retainer); - WalFile(std::filesystem::path current_wal_path, Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num, - uint64_t from_timestamp, uint64_t to_timestamp, uint64_t count, utils::FileRetainer *file_retainer); - - WalFile(const WalFile &) = delete; - WalFile(WalFile &&) = delete; - WalFile &operator=(const WalFile &) = delete; - WalFile &operator=(WalFile &&) = delete; - - ~WalFile(); - - void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp); - void AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp); - - void AppendTransactionEnd(uint64_t timestamp); - - void AppendOperation(StorageGlobalOperation operation, LabelId label, const std::set &properties, - uint64_t timestamp); - - void Sync(); - - uint64_t GetSize(); - - uint64_t SequenceNumber() const; - - auto FromTimestamp() const { return from_timestamp_; } - - auto ToTimestamp() const { return to_timestamp_; } - - auto Count() const { return count_; } - - // Disable flushing of the internal buffer. - void DisableFlushing(); - // Enable flushing of the internal buffer. - void EnableFlushing(); - // Try flushing the internal buffer. - void TryFlushing(); - // Get the internal buffer with its size. - std::pair CurrentFileBuffer() const; - - // Get the path of the current WAL file. - const auto &Path() const { return path_; } - - void FinalizeWal(); - void DeleteWal(); - - private: - void UpdateStats(uint64_t timestamp); - - Config::Items items_; - NameIdMapper *name_id_mapper_; - Encoder wal_; - std::filesystem::path path_; - uint64_t from_timestamp_; - uint64_t to_timestamp_; - uint64_t count_; - uint64_t seq_num_; - - utils::FileRetainer *file_retainer_; -}; - -} // namespace memgraph::storage::v3::durability diff --git a/src/storage/v3/edge_accessor.hpp b/src/storage/v3/edge_accessor.hpp index 98a3286d8..546a09200 100644 --- a/src/storage/v3/edge_accessor.hpp +++ b/src/storage/v3/edge_accessor.hpp @@ -28,7 +28,6 @@ namespace memgraph::storage::v3 { struct Vertex; class VertexAccessor; struct Indices; -struct Constraints; class EdgeAccessor final { private: @@ -36,14 +35,13 @@ class EdgeAccessor final { public: EdgeAccessor(EdgeRef edge, EdgeTypeId edge_type, VertexId from_vertex, VertexId to_vertex, Transaction *transaction, - Indices *indices, Constraints *constraints, Config::Items config, bool for_deleted = false) + Indices *indices, Config::Items config, bool for_deleted = false) : edge_(edge), edge_type_(edge_type), from_vertex_(std::move(from_vertex)), to_vertex_(std::move(to_vertex)), transaction_(transaction), indices_(indices), - constraints_(constraints), config_(config), for_deleted_(for_deleted) {} @@ -93,7 +91,6 @@ class EdgeAccessor final { VertexId to_vertex_; Transaction *transaction_; Indices *indices_; - Constraints *constraints_; Config::Items config_; // if the accessor was created for a deleted edge. diff --git a/src/storage/v3/indices.cpp b/src/storage/v3/indices.cpp index 82e7eadf2..b6a01b81a 100644 --- a/src/storage/v3/indices.cpp +++ b/src/storage/v3/indices.cpp @@ -32,16 +32,16 @@ namespace { template bool AnyVersionSatisfiesPredicate(uint64_t timestamp, const Delta *delta, const TCallback &predicate) { while (delta != nullptr) { - auto ts = delta->timestamp->load(std::memory_order_acquire); + const auto commit_info = *delta->commit_info; // This is a committed change that we see so we shouldn't undo it. - if (ts < timestamp) { + if (commit_info.is_locally_committed && commit_info.start_or_commit_timestamp.logical_id < timestamp) { break; } if (predicate(*delta)) { return true; } // Move to the next delta. - delta = delta->next.load(std::memory_order_acquire); + delta = delta->next; } return false; } @@ -266,7 +266,7 @@ void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transacti auto it = index_.find(label); if (it == index_.end()) return; auto acc = it->second.access(); - acc.insert(Entry{vertex, tx.start_timestamp}); + acc.insert(Entry{vertex, tx.start_timestamp.logical_id}); } bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) { @@ -302,20 +302,20 @@ std::vector LabelIndex::ListIndices() const { return ret; } -void LabelIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) { +void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) { for (auto &label_storage : index_) { auto vertices_acc = label_storage.second.access(); for (auto it = vertices_acc.begin(); it != vertices_acc.end();) { auto next_it = it; ++next_it; - if (it->timestamp >= oldest_active_start_timestamp) { + if (it->timestamp >= clean_up_before_timestamp) { it = next_it; continue; } if ((next_it != vertices_acc.end() && it->vertex == next_it->vertex) || - !AnyVersionHasLabel(*it->vertex, label_storage.first, oldest_active_start_timestamp)) { + !AnyVersionHasLabel(*it->vertex, label_storage.first, clean_up_before_timestamp)) { vertices_acc.remove(*it); } @@ -327,7 +327,7 @@ void LabelIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) { LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList::Iterator index_iterator) : self_(self), index_iterator_(index_iterator), - current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_), + current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_), current_vertex_(nullptr) { AdvanceUntilValid(); } @@ -345,22 +345,21 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() { } if (CurrentVersionHasLabel(*index_iterator_->vertex, self_->label_, self_->transaction_, self_->view_)) { current_vertex_ = index_iterator_->vertex; - current_vertex_accessor_ = VertexAccessor{current_vertex_, self_->transaction_, self_->indices_, - self_->constraints_, self_->config_, *self_->vertex_validator_}; + current_vertex_accessor_ = VertexAccessor{current_vertex_, self_->transaction_, self_->indices_, self_->config_, + *self_->vertex_validator_}; break; } } } LabelIndex::Iterable::Iterable(utils::SkipList::Accessor index_accessor, LabelId label, View view, - Transaction *transaction, Indices *indices, Constraints *constraints, - Config::Items config, const VertexValidator &vertex_validator) + Transaction *transaction, Indices *indices, Config::Items config, + const VertexValidator &vertex_validator) : index_accessor_(std::move(index_accessor)), label_(label), view_(view), transaction_(transaction), indices_(indices), - constraints_(constraints), config_(config), vertex_validator_(&vertex_validator) {} @@ -396,7 +395,7 @@ void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const T auto prop_value = vertex->properties.GetProperty(label_prop.second); if (!prop_value.IsNull()) { auto acc = storage.access(); - acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp}); + acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp.logical_id}); } } } @@ -412,7 +411,7 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property } if (utils::Contains(vertex->labels, label_prop.first)) { auto acc = storage.access(); - acc.insert(Entry{value, vertex, tx.start_timestamp}); + acc.insert(Entry{value, vertex, tx.start_timestamp.logical_id}); } } } @@ -455,21 +454,21 @@ std::vector> LabelPropertyIndex::ListIndices() co return ret; } -void LabelPropertyIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) { +void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) { for (auto &[label_property, index] : index_) { auto index_acc = index.access(); for (auto it = index_acc.begin(); it != index_acc.end();) { auto next_it = it; ++next_it; - if (it->timestamp >= oldest_active_start_timestamp) { + if (it->timestamp >= clean_up_before_timestamp) { it = next_it; continue; } if ((next_it != index_acc.end() && it->vertex == next_it->vertex && it->value == next_it->value) || !AnyVersionHasLabelProperty(*it->vertex, label_property.first, label_property.second, it->value, - oldest_active_start_timestamp)) { + clean_up_before_timestamp)) { index_acc.remove(*it); } it = next_it; @@ -480,7 +479,7 @@ void LabelPropertyIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_time LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList::Iterator index_iterator) : self_(self), index_iterator_(index_iterator), - current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_), + current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_), current_vertex_(nullptr) { AdvanceUntilValid(); } @@ -519,8 +518,8 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() { if (CurrentVersionHasLabelProperty(*index_iterator_->vertex, self_->label_, self_->property_, index_iterator_->value, self_->transaction_, self_->view_)) { current_vertex_ = index_iterator_->vertex; - current_vertex_accessor_ = VertexAccessor(current_vertex_, self_->transaction_, self_->indices_, - self_->constraints_, self_->config_, *self_->vertex_validator_); + current_vertex_accessor_ = VertexAccessor(current_vertex_, self_->transaction_, self_->indices_, self_->config_, + *self_->vertex_validator_); break; } } @@ -542,8 +541,8 @@ LabelPropertyIndex::Iterable::Iterable(utils::SkipList::Accessor index_ac PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view, - Transaction *transaction, Indices *indices, Constraints *constraints, - Config::Items config, const VertexValidator &vertex_validator) + Transaction *transaction, Indices *indices, Config::Items config, + const VertexValidator &vertex_validator) : index_accessor_(std::move(index_accessor)), label_(label), property_(property), @@ -552,7 +551,6 @@ LabelPropertyIndex::Iterable::Iterable(utils::SkipList::Accessor index_ac view_(view), transaction_(transaction), indices_(indices), - constraints_(constraints), config_(config), vertex_validator_(&vertex_validator) { // We have to fix the bounds that the user provided to us. If the user @@ -698,9 +696,9 @@ void LabelPropertyIndex::RunGC() { } } -void RemoveObsoleteEntries(Indices *indices, uint64_t oldest_active_start_timestamp) { - indices->label_index.RemoveObsoleteEntries(oldest_active_start_timestamp); - indices->label_property_index.RemoveObsoleteEntries(oldest_active_start_timestamp); +void RemoveObsoleteEntries(Indices *indices, const uint64_t clean_up_before_timestamp) { + indices->label_index.RemoveObsoleteEntries(clean_up_before_timestamp); + indices->label_property_index.RemoveObsoleteEntries(clean_up_before_timestamp); } void UpdateOnAddLabel(Indices *indices, LabelId label, Vertex *vertex, const Transaction &tx) { diff --git a/src/storage/v3/indices.hpp b/src/storage/v3/indices.hpp index 622c46347..0dd09ce53 100644 --- a/src/storage/v3/indices.hpp +++ b/src/storage/v3/indices.hpp @@ -28,7 +28,6 @@ namespace memgraph::storage::v3 { struct Indices; -struct Constraints; class LabelIndex { private: @@ -53,8 +52,8 @@ class LabelIndex { }; public: - LabelIndex(Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator) - : indices_(indices), constraints_(constraints), config_(config), vertex_validator_{&vertex_validator} {} + LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator) + : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {} /// @throw std::bad_alloc void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx); @@ -69,12 +68,12 @@ class LabelIndex { std::vector ListIndices() const; - void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp); + void RemoveObsoleteEntries(uint64_t clean_up_before_timestamp); class Iterable { public: Iterable(utils::SkipList::Accessor index_accessor, LabelId label, View view, Transaction *transaction, - Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator); + Indices *indices, Config::Items config, const VertexValidator &vertex_validator); class Iterator { public: @@ -105,7 +104,6 @@ class LabelIndex { View view_; Transaction *transaction_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; }; @@ -114,7 +112,7 @@ class LabelIndex { Iterable Vertices(LabelId label, View view, Transaction *transaction) { auto it = index_.find(label); MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint()); - return {it->second.access(), label, view, transaction, indices_, constraints_, config_, *vertex_validator_}; + return {it->second.access(), label, view, transaction, indices_, config_, *vertex_validator_}; } int64_t ApproximateVertexCount(LabelId label) { @@ -130,7 +128,6 @@ class LabelIndex { private: std::map> index_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; }; @@ -150,9 +147,8 @@ class LabelPropertyIndex { }; public: - LabelPropertyIndex(Indices *indices, Constraints *constraints, Config::Items config, - const VertexValidator &vertex_validator) - : indices_(indices), constraints_(constraints), config_(config), vertex_validator_{&vertex_validator} {} + LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator) + : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {} /// @throw std::bad_alloc void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx); @@ -169,14 +165,14 @@ class LabelPropertyIndex { std::vector> ListIndices() const; - void RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp); + void RemoveObsoleteEntries(uint64_t clean_up_before_timestamp); class Iterable { public: Iterable(utils::SkipList::Accessor index_accessor, LabelId label, PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view, Transaction *transaction, - Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator); + Indices *indices, Config::Items config, const VertexValidator &vertex_validator); class Iterator { public: @@ -211,7 +207,6 @@ class LabelPropertyIndex { View view_; Transaction *transaction_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; }; @@ -222,8 +217,8 @@ class LabelPropertyIndex { auto it = index_.find({label, property}); MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint()); - return {it->second.access(), label, property, lower_bound, upper_bound, view, - transaction, indices_, constraints_, config_, *vertex_validator_}; + return {it->second.access(), label, property, lower_bound, upper_bound, view, + transaction, indices_, config_, *vertex_validator_}; } int64_t ApproximateVertexCount(LabelId label, PropertyId property) const { @@ -250,15 +245,13 @@ class LabelPropertyIndex { private: std::map, utils::SkipList> index_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; }; struct Indices { - Indices(Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator) - : label_index(this, constraints, config, vertex_validator), - label_property_index(this, constraints, config, vertex_validator) {} + Indices(Config::Items config, const VertexValidator &vertex_validator) + : label_index(this, config, vertex_validator), label_property_index(this, config, vertex_validator) {} // Disable copy and move because members hold pointer to `this`. Indices(const Indices &) = delete; @@ -273,7 +266,7 @@ struct Indices { /// This function should be called from garbage collection to clean-up the /// index. -void RemoveObsoleteEntries(Indices *indices, uint64_t oldest_active_start_timestamp); +void RemoveObsoleteEntries(Indices *indices, uint64_t clean_up_before_timestamp); // Indices are updated whenever an update occurs, instead of only on commit or // advance command. This is necessary because we want indices to support `NEW` diff --git a/src/storage/v3/mvcc.hpp b/src/storage/v3/mvcc.hpp index ac639ffa4..219491635 100644 --- a/src/storage/v3/mvcc.hpp +++ b/src/storage/v3/mvcc.hpp @@ -28,11 +28,9 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie // if the transaction is not committed, then its deltas have transaction_id for the timestamp, otherwise they have // its commit timestamp set. // This allows the transaction to see its changes even though it's committed. - const auto commit_timestamp = transaction->commit_timestamp - ? transaction->commit_timestamp->load(std::memory_order_acquire) - : transaction->transaction_id; + const auto &commit_info = *transaction->commit_info; while (delta != nullptr) { - auto ts = delta->timestamp->load(std::memory_order_acquire); + const auto &delta_commit_info = *delta->commit_info; auto cid = delta->command_id; // For SNAPSHOT ISOLATION -> we can only see the changes which were committed before the start of the current @@ -44,21 +42,24 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie // id value, that the change is committed. // // For READ UNCOMMITTED -> we accept any change. - if ((transaction->isolation_level == IsolationLevel::SNAPSHOT_ISOLATION && ts < transaction->start_timestamp) || - (transaction->isolation_level == IsolationLevel::READ_COMMITTED && ts < kTransactionInitialId) || + if ((transaction->isolation_level == IsolationLevel::SNAPSHOT_ISOLATION && delta_commit_info.is_locally_committed && + delta_commit_info.start_or_commit_timestamp.logical_id < transaction->start_timestamp.logical_id) || + (transaction->isolation_level == IsolationLevel::READ_COMMITTED && delta_commit_info.is_locally_committed) || (transaction->isolation_level == IsolationLevel::READ_UNCOMMITTED)) { break; } // We shouldn't undo our newest changes because the user requested a NEW // view of the database. - if (view == View::NEW && ts == commit_timestamp && cid <= transaction->command_id) { + if (view == View::NEW && delta_commit_info.start_or_commit_timestamp == commit_info.start_or_commit_timestamp && + cid <= transaction->command_id) { break; } // We shouldn't undo our older changes because the user requested a OLD view // of the database. - if (view == View::OLD && ts == commit_timestamp && cid < transaction->command_id) { + if (view == View::OLD && delta_commit_info.start_or_commit_timestamp == commit_info.start_or_commit_timestamp && + delta->command_id < transaction->command_id) { break; } @@ -66,7 +67,7 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie callback(*delta); // Move to the next delta. - delta = delta->next.load(std::memory_order_acquire); + delta = delta->next; } } @@ -79,8 +80,10 @@ template inline bool PrepareForWrite(Transaction *transaction, TObj *object) { if (object->delta == nullptr) return true; - auto ts = object->delta->timestamp->load(std::memory_order_acquire); - if (ts == transaction->transaction_id || ts < transaction->start_timestamp) { + const auto &delta_commit_info = *object->delta->commit_info; + if (delta_commit_info.start_or_commit_timestamp == transaction->commit_info->start_or_commit_timestamp || + (delta_commit_info.is_locally_committed && + delta_commit_info.start_or_commit_timestamp < transaction->start_timestamp)) { return true; } @@ -94,8 +97,7 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) { /// a `DELETE_OBJECT` delta). /// @throw std::bad_alloc inline Delta *CreateDeleteObjectDelta(Transaction *transaction) { - transaction->EnsureCommitTimestampExists(); - return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(), + return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(), transaction->command_id); } @@ -104,8 +106,7 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) { /// @throw std::bad_alloc template inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) { - transaction->EnsureCommitTimestampExists(); - auto delta = &transaction->deltas.emplace_back(std::forward(args)..., transaction->commit_timestamp.get(), + auto delta = &transaction->deltas.emplace_back(std::forward(args)..., transaction->commit_info.get(), transaction->command_id); // The operations are written in such order so that both `next` and `prev` @@ -117,7 +118,7 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&.. // TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in // `Shard::Accessor::CreateEdge`. // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - delta->next.store(object->delta, std::memory_order_release); + delta->next = object->delta; // 2. We need to set the previous delta of the new delta to the object. delta->prev.Set(object); // 3. We need to set the previous delta of the existing delta to the new diff --git a/src/storage/v3/replication/.gitignore b/src/storage/v3/replication/.gitignore deleted file mode 100644 index 8fb0c720c..000000000 --- a/src/storage/v3/replication/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# autogenerated files -rpc.hpp diff --git a/src/storage/v3/replication/config.hpp b/src/storage/v3/replication/config.hpp deleted file mode 100644 index 093ad19ac..000000000 --- a/src/storage/v3/replication/config.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include - -namespace memgraph::storage::v3::replication { -struct ReplicationClientConfig { - std::optional timeout; - // The default delay between main checking/pinging replicas is 1s because - // that seems like a reasonable timeframe in which main should notice a - // replica is down. - std::chrono::seconds replica_check_frequency{1}; - - struct SSL { - std::string key_file; - std::string cert_file; - }; - - std::optional ssl; -}; - -struct ReplicationServerConfig { - struct SSL { - std::string key_file; - std::string cert_file; - std::string ca_file; - bool verify_peer; - }; - - std::optional ssl; -}; -} // namespace memgraph::storage::v3::replication diff --git a/src/storage/v3/replication/enums.hpp b/src/storage/v3/replication/enums.hpp deleted file mode 100644 index cccf33195..000000000 --- a/src/storage/v3/replication/enums.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once -#include - -namespace memgraph::storage::v3::replication { -enum class ReplicationMode : std::uint8_t { SYNC, ASYNC }; - -enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID }; -} // namespace memgraph::storage::v3::replication diff --git a/src/storage/v3/replication/replication_client.cpp b/src/storage/v3/replication/replication_client.cpp deleted file mode 100644 index 9a5594c5b..000000000 --- a/src/storage/v3/replication/replication_client.cpp +++ /dev/null @@ -1,617 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/replication/replication_client.hpp" - -#include -#include - -#include "storage/v3/durability/durability.hpp" -#include "storage/v3/replication/config.hpp" -#include "storage/v3/replication/enums.hpp" -#include "storage/v3/transaction.hpp" -#include "utils/file_locker.hpp" -#include "utils/logging.hpp" -#include "utils/message.hpp" - -namespace memgraph::storage::v3 { - -namespace { -template -[[maybe_unused]] inline constexpr bool always_false_v = false; -} // namespace - -////// ReplicationClient ////// -Shard::ReplicationClient::ReplicationClient(std::string name, Shard *shard, const io::network::Endpoint &endpoint, - const replication::ReplicationMode mode, - const replication::ReplicationClientConfig &config) - : name_(std::move(name)), shard_(shard), mode_(mode) { - if (config.ssl) { - rpc_context_.emplace(config.ssl->key_file, config.ssl->cert_file); - } else { - rpc_context_.emplace(); - } - - rpc_client_.emplace(endpoint, &*rpc_context_); - TryInitializeClientSync(); - - if (config.timeout && replica_state_ != replication::ReplicaState::INVALID) { - timeout_.emplace(*config.timeout); - timeout_dispatcher_.emplace(); - } - - // Help the user to get the most accurate replica state possible. - if (config.replica_check_frequency > std::chrono::seconds(0)) { - replica_checker_.Run("Replica Checker", config.replica_check_frequency, [&] { FrequentCheck(); }); - } -} - -void Shard::ReplicationClient::TryInitializeClientAsync() { - thread_pool_.AddTask([this] { - rpc_client_->Abort(); - this->TryInitializeClientSync(); - }); -} - -void Shard::ReplicationClient::FrequentCheck() { - const auto is_success = std::invoke([this]() { - try { - auto stream{rpc_client_->Stream()}; - const auto response = stream.AwaitResponse(); - return response.success; - } catch (const rpc::RpcFailedException &) { - return false; - } - }); - // States: READY, REPLICATING, RECOVERY, INVALID - // If success && ready, replicating, recovery -> stay the same because something good is going on. - // If success && INVALID -> [it's possible that replica came back to life] -> TryInitializeClient. - // If fail -> [replica is not reachable at all] -> INVALID state. - // NOTE: TryInitializeClient might return nothing if there is a branching point. - // NOTE: The early return pattern simplified the code, but the behavior should be as explained. - if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - return; - } - if (replica_state_.load() == replication::ReplicaState::INVALID) { - TryInitializeClientAsync(); - } -} - -/// @throws rpc::RpcFailedException -void Shard::ReplicationClient::InitializeClient() { - uint64_t current_commit_timestamp{kTimestampInitialId}; - - auto stream{rpc_client_->Stream(shard_->last_commit_timestamp_, shard_->epoch_id_)}; - - const auto response = stream.AwaitResponse(); - std::optional branching_point; - if (response.epoch_id != shard_->epoch_id_ && response.current_commit_timestamp != kTimestampInitialId) { - const auto &epoch_history = shard_->epoch_history_; - const auto epoch_info_iter = - std::find_if(epoch_history.crbegin(), epoch_history.crend(), - [&](const auto &epoch_info) { return epoch_info.first == response.epoch_id; }); - if (epoch_info_iter == epoch_history.crend()) { - branching_point = 0; - } else if (epoch_info_iter->second != response.current_commit_timestamp) { - branching_point = epoch_info_iter->second; - } - } - if (branching_point) { - spdlog::error( - "Replica {} cannot be used with this instance. Please start a clean " - "instance of Memgraph server on the specified endpoint.", - name_); - return; - } - - current_commit_timestamp = response.current_commit_timestamp; - spdlog::trace("Current timestamp on replica: {}", current_commit_timestamp); - spdlog::trace("Current timestamp on main: {}", shard_->last_commit_timestamp_); - if (current_commit_timestamp == shard_->last_commit_timestamp_) { - spdlog::debug("Replica '{}' up to date", name_); - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::READY); - } else { - spdlog::debug("Replica '{}' is behind", name_); - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::RECOVERY); - } - thread_pool_.AddTask([=, this] { this->RecoverReplica(current_commit_timestamp); }); - } -} - -void Shard::ReplicationClient::TryInitializeClientSync() { - try { - InitializeClient(); - } catch (const rpc::RpcFailedException &) { - std::unique_lock client_guarde{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_, - rpc_client_->Endpoint(), "https://memgr.ph/replication")); - } -} - -void Shard::ReplicationClient::HandleRpcFailure() { - spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication")); - TryInitializeClientAsync(); -} - -replication::SnapshotRes Shard::ReplicationClient::TransferSnapshot(const std::filesystem::path &path) { - auto stream{rpc_client_->Stream()}; - replication::Encoder encoder(stream.GetBuilder()); - encoder.WriteFile(path); - return stream.AwaitResponse(); -} - -replication::WalFilesRes Shard::ReplicationClient::TransferWalFiles( - const std::vector &wal_files) { - MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); - auto stream{rpc_client_->Stream(wal_files.size())}; - replication::Encoder encoder(stream.GetBuilder()); - for (const auto &wal : wal_files) { - spdlog::debug("Sending wal file: {}", wal); - encoder.WriteFile(wal); - } - - return stream.AwaitResponse(); -} - -void Shard::ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) { - std::unique_lock guard(client_lock_); - const auto status = replica_state_.load(); - switch (status) { - case replication::ReplicaState::RECOVERY: - spdlog::debug("Replica {} is behind MAIN instance", name_); - return; - case replication::ReplicaState::REPLICATING: - spdlog::debug("Replica {} missed a transaction", name_); - // We missed a transaction because we're still replicating - // the previous transaction so we need to go to RECOVERY - // state to catch up with the missing transaction - // We cannot queue the recovery process here because - // an error can happen while we're replicating the previous - // transaction after which the client should go to - // INVALID state before starting the recovery process - replica_state_.store(replication::ReplicaState::RECOVERY); - return; - case replication::ReplicaState::INVALID: - HandleRpcFailure(); - return; - case replication::ReplicaState::READY: - MG_ASSERT(!replica_stream_); - try { - replica_stream_.emplace(ReplicaStream{this, shard_->last_commit_timestamp_, current_wal_seq_num}); - replica_state_.store(replication::ReplicaState::REPLICATING); - } catch (const rpc::RpcFailedException &) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); - } - return; - } -} - -void Shard::ReplicationClient::IfStreamingTransaction(const std::function &callback) { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return; - } - - try { - callback(*replica_stream_); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - } -} - -void Shard::ReplicationClient::FinalizeTransactionReplication() { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return; - } - - if (mode_ == replication::ReplicationMode::ASYNC) { - thread_pool_.AddTask([this] { this->FinalizeTransactionReplicationInternal(); }); - } else if (timeout_) { - MG_ASSERT(mode_ == replication::ReplicationMode::SYNC, "Only SYNC replica can have a timeout."); - MG_ASSERT(timeout_dispatcher_, "Timeout thread is missing"); - timeout_dispatcher_->WaitForTaskToFinish(); - - timeout_dispatcher_->active = true; - thread_pool_.AddTask([&, this] { - this->FinalizeTransactionReplicationInternal(); - std::unique_lock main_guard(timeout_dispatcher_->main_lock); - // TimerThread can finish waiting for timeout - timeout_dispatcher_->active = false; - // Notify the main thread - timeout_dispatcher_->main_cv.notify_one(); - }); - - timeout_dispatcher_->StartTimeoutTask(*timeout_); - - // Wait until one of the threads notifies us that they finished executing - // Both threads should first set the active flag to false - { - std::unique_lock main_guard(timeout_dispatcher_->main_lock); - timeout_dispatcher_->main_cv.wait(main_guard, [&] { return !timeout_dispatcher_->active.load(); }); - } - - // TODO (antonio2368): Document and/or polish SEMI-SYNC to ASYNC fallback. - if (replica_state_ == replication::ReplicaState::REPLICATING) { - mode_ = replication::ReplicationMode::ASYNC; - timeout_.reset(); - // This can only happen if we timeouted so we are sure that - // Timeout task finished - // We need to delete timeout dispatcher AFTER the replication - // finished because it tries to acquire the timeout lock - // and acces the `active` variable` - thread_pool_.AddTask([this] { timeout_dispatcher_.reset(); }); - } - } else { - FinalizeTransactionReplicationInternal(); - } -} - -void Shard::ReplicationClient::FinalizeTransactionReplicationInternal() { - MG_ASSERT(replica_stream_, "Missing stream for transaction deltas"); - try { - auto response = replica_stream_->Finalize(); - replica_stream_.reset(); - std::unique_lock client_guard(client_lock_); - if (!response.success || replica_state_ == replication::ReplicaState::RECOVERY) { - replica_state_.store(replication::ReplicaState::RECOVERY); - thread_pool_.AddTask([&, this] { this->RecoverReplica(response.current_commit_timestamp); }); - } else { - replica_state_.store(replication::ReplicaState::READY); - } - } catch (const rpc::RpcFailedException &) { - replica_stream_.reset(); - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - } -} - -void Shard::ReplicationClient::RecoverReplica(uint64_t replica_commit) { - while (true) { - auto file_locker = shard_->file_retainer_.AddLocker(); - - const auto steps = GetRecoverySteps(replica_commit, &file_locker); - for (const auto &recovery_step : steps) { - try { - std::visit( - [&, this](T &&arg) { - using StepType = std::remove_cvref_t; - if constexpr (std::is_same_v) { - spdlog::debug("Sending the latest snapshot file: {}", arg); - auto response = TransferSnapshot(arg); - replica_commit = response.current_commit_timestamp; - } else if constexpr (std::is_same_v) { - spdlog::debug("Sending the latest wal files"); - auto response = TransferWalFiles(arg); - replica_commit = response.current_commit_timestamp; - } else if constexpr (std::is_same_v) { - if (shard_->wal_file_ && shard_->wal_file_->SequenceNumber() == arg.current_wal_seq_num) { - shard_->wal_file_->DisableFlushing(); - spdlog::debug("Sending current wal file"); - replica_commit = ReplicateCurrentWal(); - shard_->wal_file_->EnableFlushing(); - } - } else { - static_assert(always_false_v, "Missing type from variant visitor"); - } - }, - recovery_step); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - return; - } - } - - spdlog::trace("Current timestamp on replica: {}", replica_commit); - // To avoid the situation where we read a correct commit timestamp in - // one thread, and after that another thread commits a different a - // transaction and THEN we set the state to READY in the first thread, - // we set this lock before checking the timestamp. - // We will detect that the state is invalid during the next commit, - // because replication::AppendDeltasRpc sends the last commit timestamp which - // replica checks if it's the same last commit timestamp it received - // and we will go to recovery. - // By adding this lock, we can avoid that, and go to RECOVERY immediately. - std::unique_lock client_guard{client_lock_}; - SPDLOG_INFO("Replica timestamp: {}", replica_commit); - SPDLOG_INFO("Last commit: {}", shard_->last_commit_timestamp_); - if (shard_->last_commit_timestamp_ == replica_commit) { - replica_state_.store(replication::ReplicaState::READY); - return; - } - } -} - -uint64_t Shard::ReplicationClient::ReplicateCurrentWal() { - const auto &wal_file = shard_->wal_file_; - auto stream = TransferCurrentWalFile(); - stream.AppendFilename(wal_file->Path().filename()); - utils::InputFile file; - MG_ASSERT(file.Open(shard_->wal_file_->Path()), "Failed to open current WAL file!"); - const auto [buffer, buffer_size] = wal_file->CurrentFileBuffer(); - stream.AppendSize(file.GetSize() + buffer_size); - stream.AppendFileData(&file); - stream.AppendBufferData(buffer, buffer_size); - auto response = stream.Finalize(); - return response.current_commit_timestamp; -} - -/// This method tries to find the optimal path for recoverying a single replica. -/// Based on the last commit transfered to replica it tries to update the -/// replica using durability files - WALs and Snapshots. WAL files are much -/// smaller in size as they contain only the Deltas (changes) made during the -/// transactions while Snapshots contain all the data. For that reason we prefer -/// WALs as much as possible. As the WAL file that is currently being updated -/// can change during the process we ignore it as much as possible. Also, it -/// uses the transaction lock so lokcing it can be really expensive. After we -/// fetch the list of finalized WALs, we try to find the longest chain of -/// sequential WALs, starting from the latest one, that will update the recovery -/// with the all missed updates. If the WAL chain cannot be created, replica is -/// behind by a lot, so we use the regular recovery process, we send the latest -/// snapshot and all the necessary WAL files, starting from the newest WAL that -/// contains a timestamp before the snapshot. If we registered the existence of -/// the current WAL, we add the sequence number we read from it to the recovery -/// process. After all the other steps are finished, if the current WAL contains -/// the same sequence number, it's the same WAL we read while fetching the -/// recovery steps, so we can safely send it to the replica. -/// We assume that the property of preserving at least 1 WAL before the snapshot -/// is satisfied as we extract the timestamp information from it. -std::vector Shard::ReplicationClient::GetRecoverySteps( - const uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker) { - // First check if we can recover using the current wal file only - // otherwise save the seq_num of the current wal file - // This lock is also necessary to force the missed transaction to finish. - std::optional current_wal_seq_num; - std::optional current_wal_from_timestamp; - if (shard_->wal_file_) { - current_wal_seq_num.emplace(shard_->wal_file_->SequenceNumber()); - current_wal_from_timestamp.emplace(shard_->wal_file_->FromTimestamp()); - } - - auto locker_acc = file_locker->Access(); - auto wal_files = durability::GetWalFiles(shard_->wal_directory_, shard_->uuid_, current_wal_seq_num); - MG_ASSERT(wal_files, "Wal files could not be loaded"); - - auto snapshot_files = durability::GetSnapshotFiles(shard_->snapshot_directory_, shard_->uuid_); - std::optional latest_snapshot; - if (!snapshot_files.empty()) { - std::sort(snapshot_files.begin(), snapshot_files.end()); - latest_snapshot.emplace(std::move(snapshot_files.back())); - } - - std::vector recovery_steps; - - // No finalized WAL files were found. This means the difference is contained - // inside the current WAL or the snapshot. - if (wal_files->empty()) { - if (current_wal_from_timestamp && replica_commit >= *current_wal_from_timestamp) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Without the finalized WAL containing the current timestamp of replica, - // we cannot know if the difference is only in the current WAL or we need - // to send the snapshot. - if (latest_snapshot) { - locker_acc.AddPath(latest_snapshot->path); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - } - // if there are no finalized WAL files, snapshot left the current WAL - // as the WAL file containing a transaction before snapshot creation - // so we can be sure that the current WAL is present - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Find the longest chain of WALs for recovery. - // The chain consists ONLY of sequential WALs. - auto rwal_it = wal_files->rbegin(); - - // if the last finalized WAL is before the replica commit - // then we can recovery only from current WAL - if (rwal_it->to_timestamp <= replica_commit) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - uint64_t previous_seq_num{rwal_it->seq_num}; - for (; rwal_it != wal_files->rend(); ++rwal_it) { - // If the difference between two consecutive wal files is not 0 or 1 - // we have a missing WAL in our chain - if (previous_seq_num - rwal_it->seq_num > 1) { - break; - } - - // Find first WAL that contains up to replica commit, i.e. WAL - // that is before the replica commit or conatins the replica commit - // as the last committed transaction OR we managed to find the first WAL - // file. - if (replica_commit >= rwal_it->from_timestamp || rwal_it->seq_num == 0) { - if (replica_commit >= rwal_it->to_timestamp) { - // We want the WAL after because the replica already contains all the - // commits from this WAL - --rwal_it; - } - std::vector wal_chain; - auto distance_from_first = std::distance(rwal_it, wal_files->rend() - 1); - // We have managed to create WAL chain - // We need to lock these files and add them to the chain - for (auto result_wal_it = wal_files->begin() + distance_from_first; result_wal_it != wal_files->end(); - ++result_wal_it) { - locker_acc.AddPath(result_wal_it->path); - wal_chain.push_back(std::move(result_wal_it->path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(wal_chain)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - return recovery_steps; - } - - previous_seq_num = rwal_it->seq_num; - } - - MG_ASSERT(latest_snapshot, "Invalid durability state, missing snapshot"); - // We didn't manage to find a WAL chain, we need to send the latest snapshot - // with its WALs - locker_acc.AddPath(latest_snapshot->path); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - - std::vector recovery_wal_files; - auto wal_it = wal_files->begin(); - for (; wal_it != wal_files->end(); ++wal_it) { - // Assuming recovery process is correct the snashpot should - // always retain a single WAL that contains a transaction - // before its creation - if (latest_snapshot->start_timestamp < wal_it->to_timestamp) { - if (latest_snapshot->start_timestamp < wal_it->from_timestamp) { - MG_ASSERT(wal_it != wal_files->begin(), "Invalid durability files state"); - --wal_it; - } - break; - } - } - - for (; wal_it != wal_files->end(); ++wal_it) { - locker_acc.AddPath(wal_it->path); - recovery_wal_files.push_back(std::move(wal_it->path)); - } - - // We only have a WAL before the snapshot - if (recovery_wal_files.empty()) { - locker_acc.AddPath(wal_files->back().path); - recovery_wal_files.push_back(std::move(wal_files->back().path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(recovery_wal_files)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - - return recovery_steps; -} - -////// TimeoutDispatcher ////// -void Shard::ReplicationClient::TimeoutDispatcher::WaitForTaskToFinish() { - // Wait for the previous timeout task to finish - std::unique_lock main_guard(main_lock); - main_cv.wait(main_guard, [&] { return finished; }); -} - -void Shard::ReplicationClient::TimeoutDispatcher::StartTimeoutTask(const double timeout) { - timeout_pool.AddTask([timeout, this] { - finished = false; - using std::chrono::steady_clock; - const auto timeout_duration = - std::chrono::duration_cast(std::chrono::duration(timeout)); - const auto end_time = steady_clock::now() + timeout_duration; - while (active && (steady_clock::now() < end_time)) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - - std::unique_lock main_guard(main_lock); - finished = true; - active = false; - main_cv.notify_one(); - }); -} -////// ReplicaStream ////// -Shard::ReplicationClient::ReplicaStream::ReplicaStream(ReplicationClient *self, - const uint64_t previous_commit_timestamp, - const uint64_t current_seq_num) - : self_(self), - stream_(self_->rpc_client_->Stream(previous_commit_timestamp, current_seq_num)) { - replication::Encoder encoder{stream_.GetBuilder()}; - encoder.WriteString(self_->shard_->epoch_id_); -} - -void Shard::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex, - uint64_t final_commit_timestamp) { - replication::Encoder encoder(stream_.GetBuilder()); - EncodeDelta(&encoder, &self_->shard_->name_id_mapper_, self_->shard_->config_.items, delta, vertex, - final_commit_timestamp); -} - -void Shard::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge, - uint64_t final_commit_timestamp) { - replication::Encoder encoder(stream_.GetBuilder()); - EncodeDelta(&encoder, &self_->shard_->name_id_mapper_, delta, edge, final_commit_timestamp); -} - -void Shard::ReplicationClient::ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) { - replication::Encoder encoder(stream_.GetBuilder()); - EncodeTransactionEnd(&encoder, final_commit_timestamp); -} - -void Shard::ReplicationClient::ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation, - LabelId label, const std::set &properties, - uint64_t timestamp) { - replication::Encoder encoder(stream_.GetBuilder()); - EncodeOperation(&encoder, &self_->shard_->name_id_mapper_, operation, label, properties, timestamp); -} - -replication::AppendDeltasRes Shard::ReplicationClient::ReplicaStream::Finalize() { return stream_.AwaitResponse(); } - -////// CurrentWalHandler ////// -Shard::ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self) - : self_(self), stream_(self_->rpc_client_->Stream()) {} - -void Shard::ReplicationClient::CurrentWalHandler::AppendFilename(const std::string &filename) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteString(filename); -} - -void Shard::ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteUint(size); -} - -void Shard::ReplicationClient::CurrentWalHandler::AppendFileData(utils::InputFile *file) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteFileData(file); -} - -void Shard::ReplicationClient::CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteBuffer(buffer, buffer_size); -} - -replication::CurrentWalRes Shard::ReplicationClient::CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/replication/replication_client.hpp b/src/storage/v3/replication/replication_client.hpp deleted file mode 100644 index 7a1731f5f..000000000 --- a/src/storage/v3/replication/replication_client.hpp +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include -#include -#include -#include - -#include "rpc/client.hpp" -#include "storage/v3/config.hpp" -#include "storage/v3/delta.hpp" -#include "storage/v3/durability/wal.hpp" -#include "storage/v3/id_types.hpp" -#include "storage/v3/mvcc.hpp" -#include "storage/v3/name_id_mapper.hpp" -#include "storage/v3/property_value.hpp" -#include "storage/v3/replication/config.hpp" -#include "storage/v3/replication/enums.hpp" -#include "storage/v3/replication/rpc.hpp" -#include "storage/v3/replication/serialization.hpp" -#include "storage/v3/shard.hpp" -#include "utils/file.hpp" -#include "utils/file_locker.hpp" -#include "utils/spin_lock.hpp" -#include "utils/synchronized.hpp" -#include "utils/thread_pool.hpp" - -namespace memgraph::storage::v3 { - -class Shard::ReplicationClient { - public: - ReplicationClient(std::string name, Shard *shard, const io::network::Endpoint &endpoint, - replication::ReplicationMode mode, const replication::ReplicationClientConfig &config = {}); - - // Handler used for transfering the current transaction. - class ReplicaStream { - private: - friend class ReplicationClient; - explicit ReplicaStream(ReplicationClient *self, uint64_t previous_commit_timestamp, uint64_t current_seq_num); - - public: - /// @throw rpc::RpcFailedException - void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp); - - /// @throw rpc::RpcFailedException - void AppendDelta(const Delta &delta, const Edge &edge, uint64_t final_commit_timestamp); - - /// @throw rpc::RpcFailedException - void AppendTransactionEnd(uint64_t final_commit_timestamp); - - /// @throw rpc::RpcFailedException - void AppendOperation(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t timestamp); - - private: - /// @throw rpc::RpcFailedException - replication::AppendDeltasRes Finalize(); - - ReplicationClient *self_; - rpc::Client::StreamHandler stream_; - }; - - // Handler for transfering the current WAL file whose data is - // contained in the internal buffer and the file. - class CurrentWalHandler { - private: - friend class ReplicationClient; - explicit CurrentWalHandler(ReplicationClient *self); - - public: - void AppendFilename(const std::string &filename); - - void AppendSize(size_t size); - - void AppendFileData(utils::InputFile *file); - - void AppendBufferData(const uint8_t *buffer, size_t buffer_size); - - /// @throw rpc::RpcFailedException - replication::CurrentWalRes Finalize(); - - private: - ReplicationClient *self_; - rpc::Client::StreamHandler stream_; - }; - - void StartTransactionReplication(uint64_t current_wal_seq_num); - - // Replication clients can be removed at any point - // so to avoid any complexity of checking if the client was removed whenever - // we want to send part of transaction and to avoid adding some GC logic this - // function will run a callback if, after previously callling - // StartTransactionReplication, stream is created. - void IfStreamingTransaction(const std::function &callback); - - void FinalizeTransactionReplication(); - - // Transfer the snapshot file. - // @param path Path of the snapshot file. - replication::SnapshotRes TransferSnapshot(const std::filesystem::path &path); - - CurrentWalHandler TransferCurrentWalFile() { return CurrentWalHandler{this}; } - - // Transfer the WAL files - replication::WalFilesRes TransferWalFiles(const std::vector &wal_files); - - const auto &Name() const { return name_; } - - auto State() const { return replica_state_.load(); } - - auto Mode() const { return mode_; } - - auto Timeout() const { return timeout_; } - - const auto &Endpoint() const { return rpc_client_->Endpoint(); } - - private: - void FinalizeTransactionReplicationInternal(); - - void RecoverReplica(uint64_t replica_commit); - - uint64_t ReplicateCurrentWal(); - - using RecoveryWals = std::vector; - struct RecoveryCurrentWal { - uint64_t current_wal_seq_num; - - explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num) : current_wal_seq_num(current_wal_seq_num) {} - }; - using RecoverySnapshot = std::filesystem::path; - using RecoveryStep = std::variant; - - std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker); - - void FrequentCheck(); - void InitializeClient(); - void TryInitializeClientSync(); - void TryInitializeClientAsync(); - void HandleRpcFailure(); - - std::string name_; - Shard *shard_; - std::optional rpc_context_; - std::optional rpc_client_; - - std::optional replica_stream_; - replication::ReplicationMode mode_{replication::ReplicationMode::SYNC}; - - // Dispatcher class for timeout tasks - struct TimeoutDispatcher { - explicit TimeoutDispatcher(){}; - - void WaitForTaskToFinish(); - - void StartTimeoutTask(double timeout); - - // If the Timeout task should continue waiting - std::atomic active{false}; - - std::mutex main_lock; - std::condition_variable main_cv; - - private: - // if the Timeout task finished executing - bool finished{true}; - - utils::ThreadPool timeout_pool{1}; - }; - - std::optional timeout_; - std::optional timeout_dispatcher_; - - utils::SpinLock client_lock_; - // This thread pool is used for background tasks so we don't - // block the main storage thread - // We use only 1 thread for 2 reasons: - // - background tasks ALWAYS contain some kind of RPC communication. - // We can't have multiple RPC communication from a same client - // because that's not logically valid (e.g. you cannot send a snapshot - // and WAL at a same time because WAL will arrive earlier and be applied - // before the snapshot which is not correct) - // - the implementation is simplified as we have a total control of what - // this pool is executing. Also, we can simply queue multiple tasks - // and be sure of the execution order. - // Not having mulitple possible threads in the same client allows us - // to ignore concurrency problems inside the client. - utils::ThreadPool thread_pool_{1}; - std::atomic replica_state_{replication::ReplicaState::INVALID}; - - utils::Scheduler replica_checker_; -}; - -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/replication/replication_server.cpp b/src/storage/v3/replication/replication_server.cpp deleted file mode 100644 index a114c4c83..000000000 --- a/src/storage/v3/replication/replication_server.cpp +++ /dev/null @@ -1,570 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/replication/replication_server.hpp" - -#include -#include - -#include "storage/v3/durability/durability.hpp" -#include "storage/v3/durability/paths.hpp" -#include "storage/v3/durability/serialization.hpp" -#include "storage/v3/durability/snapshot.hpp" -#include "storage/v3/durability/version.hpp" -#include "storage/v3/durability/wal.hpp" -#include "storage/v3/replication/config.hpp" -#include "storage/v3/transaction.hpp" -#include "utils/exceptions.hpp" - -namespace memgraph::storage::v3 { -namespace { -std::pair ReadDelta(durability::BaseDecoder *decoder) { - try { - auto timestamp = ReadWalDeltaHeader(decoder); - SPDLOG_INFO(" Timestamp {}", timestamp); - auto delta = ReadWalDeltaData(decoder); - return {timestamp, delta}; - } catch (const slk::SlkReaderException &) { - throw utils::BasicException("Missing data!"); - } catch (const durability::RecoveryFailure &) { - throw utils::BasicException("Invalid data!"); - } -}; -} // namespace - -Shard::ReplicationServer::ReplicationServer(Shard *shard, io::network::Endpoint endpoint, - const replication::ReplicationServerConfig &config) - : shard_(shard) { - // Create RPC server. - if (config.ssl) { - rpc_server_context_.emplace(config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file, - config.ssl->verify_peer); - } else { - rpc_server_context_.emplace(); - } - // NOTE: The replication server must have a single thread for processing - // because there is no need for more processing threads - each replica can - // have only a single main server. Also, the single-threaded guarantee - // simplifies the rest of the implementation. - rpc_server_.emplace(std::move(endpoint), &*rpc_server_context_, - /* workers_count = */ 1); - - rpc_server_->Register([this](auto *req_reader, auto *res_builder) { - spdlog::debug("Received HeartbeatRpc"); - this->HeartbeatHandler(req_reader, res_builder); - }); - rpc_server_->Register([](auto *req_reader, auto *res_builder) { - spdlog::debug("Received FrequentHeartbeatRpc"); - FrequentHeartbeatHandler(req_reader, res_builder); - }); - rpc_server_->Register([this](auto *req_reader, auto *res_builder) { - spdlog::debug("Received AppendDeltasRpc"); - this->AppendDeltasHandler(req_reader, res_builder); - }); - rpc_server_->Register([this](auto *req_reader, auto *res_builder) { - spdlog::debug("Received SnapshotRpc"); - this->SnapshotHandler(req_reader, res_builder); - }); - rpc_server_->Register([this](auto *req_reader, auto *res_builder) { - spdlog::debug("Received WalFilesRpc"); - this->WalFilesHandler(req_reader, res_builder); - }); - rpc_server_->Register([this](auto *req_reader, auto *res_builder) { - spdlog::debug("Received CurrentWalRpc"); - this->CurrentWalHandler(req_reader, res_builder); - }); - rpc_server_->Start(); -} - -void Shard::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::HeartbeatReq req; - slk::Load(&req, req_reader); - replication::HeartbeatRes res{true, shard_->last_commit_timestamp_, shard_->epoch_id_}; - slk::Save(res, res_builder); -} - -void Shard::ReplicationServer::FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::FrequentHeartbeatReq req; - slk::Load(&req, req_reader); - replication::FrequentHeartbeatRes res{true}; - slk::Save(res, res_builder); -} - -void Shard::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::AppendDeltasReq req; - slk::Load(&req, req_reader); - - replication::Decoder decoder(req_reader); - - auto maybe_epoch_id = decoder.ReadString(); - MG_ASSERT(maybe_epoch_id, "Invalid replication message"); - - if (*maybe_epoch_id != shard_->epoch_id_) { - shard_->epoch_history_.emplace_back(std::move(shard_->epoch_id_), shard_->last_commit_timestamp_); - shard_->epoch_id_ = std::move(*maybe_epoch_id); - } - - if (shard_->wal_file_) { - if (req.seq_num > shard_->wal_file_->SequenceNumber() || *maybe_epoch_id != shard_->epoch_id_) { - shard_->wal_file_->FinalizeWal(); - shard_->wal_file_.reset(); - shard_->wal_seq_num_ = req.seq_num; - } else { - MG_ASSERT(shard_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file"); - shard_->wal_seq_num_ = req.seq_num + 1; - } - } else { - shard_->wal_seq_num_ = req.seq_num; - } - - if (req.previous_commit_timestamp != shard_->last_commit_timestamp_) { - // Empty the stream - bool transaction_complete = false; - while (!transaction_complete) { - SPDLOG_INFO("Skipping delta"); - const auto [timestamp, delta] = ReadDelta(&decoder); - transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type); - } - - replication::AppendDeltasRes res{false, shard_->last_commit_timestamp_}; - slk::Save(res, res_builder); - return; - } - - ReadAndApplyDelta(&decoder); - - replication::AppendDeltasRes res{true, shard_->last_commit_timestamp_}; - slk::Save(res, res_builder); -} - -void Shard::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::SnapshotReq req; - slk::Load(&req, req_reader); - - replication::Decoder decoder(req_reader); - - utils::EnsureDirOrDie(shard_->snapshot_directory_); - - const auto maybe_snapshot_path = decoder.ReadFile(shard_->snapshot_directory_); - MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!"); - spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path); - - // Clear the database - shard_->vertices_.clear(); - shard_->edges_.clear(); - - shard_->constraints_ = Constraints(); - shard_->indices_.label_index = - LabelIndex(&shard_->indices_, &shard_->constraints_, shard_->config_.items, shard_->vertex_validator_); - shard_->indices_.label_property_index = - LabelPropertyIndex(&shard_->indices_, &shard_->constraints_, shard_->config_.items, shard_->vertex_validator_); - try { - spdlog::debug("Loading snapshot"); - auto recovered_snapshot = durability::RecoveredSnapshot{}; - - // durability::LoadSnapshot(*maybe_snapshot_path, &shard_->vertices_, &shard_->edges_, - // &shard_->epoch_history_, - // &shard_->name_id_mapper_, &shard_->edge_count_, shard_->config_.items); - spdlog::debug("Snapshot loaded successfully"); - // If this step is present it should always be the first step of - // the recovery so we use the UUID we read from snasphost - shard_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid); - shard_->epoch_id_ = std::move(recovered_snapshot.snapshot_info.epoch_id); - const auto &recovery_info = recovered_snapshot.recovery_info; - shard_->timestamp_ = std::max(shard_->timestamp_, recovery_info.next_timestamp); - - // durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &shard_->indices_, - // &shard_->constraints_, &shard_->vertices_); - } catch (const durability::RecoveryFailure &e) { - LOG_FATAL("Couldn't load the snapshot because of: {}", e.what()); - } - - replication::SnapshotRes res{true, shard_->last_commit_timestamp_}; - slk::Save(res, res_builder); - - // Delete other durability files - auto snapshot_files = durability::GetSnapshotFiles(shard_->snapshot_directory_, shard_->uuid_); - for (const auto &[path, uuid, _] : snapshot_files) { - if (path != *maybe_snapshot_path) { - shard_->file_retainer_.DeleteFile(path); - } - } - - auto wal_files = durability::GetWalFiles(shard_->wal_directory_, shard_->uuid_); - if (wal_files) { - for (const auto &wal_file : *wal_files) { - shard_->file_retainer_.DeleteFile(wal_file.path); - } - - shard_->wal_file_.reset(); - } -} - -void Shard::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::WalFilesReq req; - slk::Load(&req, req_reader); - - const auto wal_file_number = req.file_number; - spdlog::debug("Received WAL files: {}", wal_file_number); - - replication::Decoder decoder(req_reader); - - utils::EnsureDirOrDie(shard_->wal_directory_); - - for (auto i = 0; i < wal_file_number; ++i) { - LoadWal(&decoder); - } - - replication::WalFilesRes res{true, shard_->last_commit_timestamp_}; - slk::Save(res, res_builder); -} - -void Shard::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - replication::CurrentWalReq req; - slk::Load(&req, req_reader); - - replication::Decoder decoder(req_reader); - - utils::EnsureDirOrDie(shard_->wal_directory_); - - LoadWal(&decoder); - - replication::CurrentWalRes res{true, shard_->last_commit_timestamp_}; - slk::Save(res, res_builder); -} - -void Shard::ReplicationServer::LoadWal(replication::Decoder *decoder) { - const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory; - utils::EnsureDir(temp_wal_directory); - auto maybe_wal_path = decoder->ReadFile(temp_wal_directory); - MG_ASSERT(maybe_wal_path, "Failed to load WAL!"); - spdlog::trace("Received WAL saved to {}", *maybe_wal_path); - try { - auto wal_info = durability::ReadWalInfo(*maybe_wal_path); - if (wal_info.seq_num == 0) { - shard_->uuid_ = wal_info.uuid; - } - - if (wal_info.epoch_id != shard_->epoch_id_) { - shard_->epoch_history_.emplace_back(wal_info.epoch_id, shard_->last_commit_timestamp_); - shard_->epoch_id_ = std::move(wal_info.epoch_id); - } - - if (shard_->wal_file_) { - if (shard_->wal_file_->SequenceNumber() != wal_info.seq_num) { - shard_->wal_file_->FinalizeWal(); - shard_->wal_seq_num_ = wal_info.seq_num; - shard_->wal_file_.reset(); - } - } else { - shard_->wal_seq_num_ = wal_info.seq_num; - } - - durability::Decoder wal; - const auto version = wal.Initialize(*maybe_wal_path, durability::kWalMagic); - if (!version) throw durability::RecoveryFailure("Couldn't read WAL magic and/or version!"); - if (!durability::IsVersionSupported(*version)) throw durability::RecoveryFailure("Invalid WAL version!"); - wal.SetPosition(wal_info.offset_deltas); - - for (size_t i = 0; i < wal_info.num_deltas;) { - i += ReadAndApplyDelta(&wal); - } - - spdlog::debug("{} loaded successfully", *maybe_wal_path); - } catch (const durability::RecoveryFailure &e) { - LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", *maybe_wal_path, e.what()); - } -} - -Shard::ReplicationServer::~ReplicationServer() { - if (rpc_server_) { - rpc_server_->Shutdown(); - rpc_server_->AwaitShutdown(); - } -} -uint64_t Shard::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *decoder) { - auto edge_acc = shard_->edges_.access(); - // auto vertex_acc = shard_->vertices_.access(); - - std::optional> commit_timestamp_and_accessor; - // auto get_transaction = [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) { - // if (!commit_timestamp_and_accessor) { - // commit_timestamp_and_accessor.emplace(commit_timestamp, shard_->Access()); - // } else if (commit_timestamp_and_accessor->first != commit_timestamp) { - // throw utils::BasicException("Received more than one transaction!"); - // } - // return &commit_timestamp_and_accessor->second; - // }; - - uint64_t applied_deltas = 0; - auto max_commit_timestamp = shard_->last_commit_timestamp_; - - for (bool transaction_complete = false; !transaction_complete; ++applied_deltas) { - const auto [timestamp, delta] = ReadDelta(decoder); - if (timestamp > max_commit_timestamp) { - max_commit_timestamp = timestamp; - } - - transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type); - - if (timestamp < shard_->timestamp_) { - continue; - } - - // SPDLOG_INFO(" Delta {}", applied_deltas); - // switch (delta.type) { - // case durability::WalDeltaData::Type::VERTEX_CREATE: { - // spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint()); - // auto *transaction = get_transaction(timestamp); - // transaction->CreateVertex(delta.vertex_create_delete.gid); - // break; - // } - // case durability::WalDeltaData::Type::VERTEX_DELETE: { - // spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint()); - // auto *transaction = get_transaction(timestamp); - // auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW); - // if (!vertex) throw utils::BasicException("Invalid transaction!"); - // auto ret = transaction->DeleteVertex(&*vertex); - // if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: { - // spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(), - // delta.vertex_add_remove_label.label); - // auto *transaction = get_transaction(timestamp); - // auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW); - // if (!vertex) throw utils::BasicException("Invalid transaction!"); - // auto ret = vertex->AddLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label)); - // if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: { - // spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(), - // delta.vertex_add_remove_label.label); - // auto *transaction = get_transaction(timestamp); - // auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW); - // if (!vertex) throw utils::BasicException("Invalid transaction!"); - // auto ret = vertex->RemoveLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label)); - // if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: { - // spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(), - // delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value); - // auto *transaction = get_transaction(timestamp); - // auto vertex = transaction->FindVertex(delta.vertex_edge_set_property.gid, View::NEW); - // if (!vertex) throw utils::BasicException("Invalid transaction!"); - // auto ret = vertex->SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property), - // delta.vertex_edge_set_property.value); - // if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::EDGE_CREATE: { - // spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}", - // delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type, - // delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); - // auto *transaction = get_transaction(timestamp); - // auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW); - // if (!from_vertex) throw utils::BasicException("Invalid transaction!"); - // auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW); - // if (!to_vertex) throw utils::BasicException("Invalid transaction!"); - // auto edge = transaction->CreateEdge(&*from_vertex, &*to_vertex, - // transaction->NameToEdgeType(delta.edge_create_delete.edge_type), - // delta.edge_create_delete.gid); - // if (edge.HasError()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::EDGE_DELETE: { - // spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}", - // delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type, - // delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); - // auto *transaction = get_transaction(timestamp); - // auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW); - // if (!from_vertex) throw utils::BasicException("Invalid transaction!"); - // auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW); - // if (!to_vertex) throw utils::BasicException("Invalid transaction!"); - // auto edges = from_vertex->OutEdges(View::NEW, - // {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)}, - // &*to_vertex); - // if (edges.HasError()) throw utils::BasicException("Invalid transaction!"); - // if (edges->size() != 1) throw utils::BasicException("Invalid transaction!"); - // auto &edge = (*edges)[0]; - // auto ret = transaction->DeleteEdge(&edge); - // if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: { - // spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(), - // delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value); - - // if (!shard_->config_.items.properties_on_edges) - // throw utils::BasicException( - // "Can't set properties on edges because properties on edges " - // "are disabled!"); - - // // auto *transaction = get_transaction(timestamp); - - // // The following block of code effectively implements `FindEdge` and - // // yields an accessor that is only valid for managing the edge's - // // properties. - // auto edge = edge_acc.find(delta.vertex_edge_set_property.gid); - // if (edge == edge_acc.end()) throw utils::BasicException("Invalid transaction!"); - // // The edge visibility check must be done here manually because we - // // don't allow direct access to the edges through the public API. - // { - // auto is_visible = !edge->deleted; - // auto *delta = edge->delta; - // ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW, [&is_visible](const Delta &delta) { - // switch (delta.action) { - // case Delta::Action::ADD_LABEL: - // case Delta::Action::REMOVE_LABEL: - // case Delta::Action::SET_PROPERTY: - // case Delta::Action::ADD_IN_EDGE: - // case Delta::Action::ADD_OUT_EDGE: - // case Delta::Action::REMOVE_IN_EDGE: - // case Delta::Action::REMOVE_OUT_EDGE: - // break; - // case Delta::Action::RECREATE_OBJECT: { - // is_visible = true; - // break; - // } - // case Delta::Action::DELETE_OBJECT: { - // is_visible = false; - // break; - // } - // } - // }); - // if (!is_visible) throw utils::BasicException("Invalid transaction!"); - // } - // EdgeRef edge_ref(&*edge); - // // Here we create an edge accessor that we will use to get the - // // properties of the edge. The accessor is created with an invalid - // // type and invalid from/to pointers because we don't know them - // // here, but that isn't an issue because we won't use that part of - // // the API here. - // auto ea = EdgeAccessor{edge_ref, - // EdgeTypeId::FromUint(0UL), - // nullptr, - // nullptr, - // &transaction->transaction_, - // &shard_->indices_, - // &shard_->constraints_, - // shard_->config_.items, - // shard_->schema_validator_}; - - // auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property), - // delta.vertex_edge_set_property.value); - // if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - - // case durability::WalDeltaData::Type::TRANSACTION_END: { - // spdlog::trace(" Transaction end"); - // if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp) - // throw utils::BasicException("Invalid data!"); - // auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first); - // if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); - // commit_timestamp_and_accessor = std::nullopt; - // break; - // } - - // case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: { - // spdlog::trace(" Create label index on :{}", delta.operation_label.label); - // // Need to send the timestamp - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // if (!shard_->CreateIndex(shard_->NameToLabel(delta.operation_label.label), timestamp)) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::LABEL_INDEX_DROP: { - // spdlog::trace(" Drop label index on :{}", delta.operation_label.label); - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // if (!shard_->DropIndex(shard_->NameToLabel(delta.operation_label.label), timestamp)) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: { - // spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label, - // delta.operation_label_property.property); - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // if (!shard_->CreateIndex(shard_->NameToLabel(delta.operation_label_property.label), - // shard_->NameToProperty(delta.operation_label_property.property), timestamp)) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: { - // spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label, - // delta.operation_label_property.property); - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // if (!shard_->DropIndex(shard_->NameToLabel(delta.operation_label_property.label), - // shard_->NameToProperty(delta.operation_label_property.property), timestamp)) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: { - // spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label, - // delta.operation_label_property.property); - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // auto ret = shard_->CreateExistenceConstraint( - // shard_->NameToLabel(delta.operation_label_property.label), - // shard_->NameToProperty(delta.operation_label_property.property), timestamp); - // if (!ret.HasValue() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { - // spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label, - // delta.operation_label_property.property); - // if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // if (!shard_->DropExistenceConstraint(shard_->NameToLabel(delta.operation_label_property.label), - // shard_->NameToProperty(delta.operation_label_property.property), - // timestamp)) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: { - // std::stringstream ss; - // utils::PrintIterable(ss, delta.operation_label_properties.properties); - // spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, - // ss.str()); if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // std::set properties; - // for (const auto &prop : delta.operation_label_properties.properties) { - // properties.emplace(shard_->NameToProperty(prop)); - // } - // auto ret = shard_->CreateUniqueConstraint(shard_->NameToLabel(delta.operation_label_properties.label), - // properties, timestamp); - // if (!ret.HasValue() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) - // throw utils::BasicException("Invalid transaction!"); - // break; - // } - // case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { - // std::stringstream ss; - // utils::PrintIterable(ss, delta.operation_label_properties.properties); - // spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, - // ss.str()); if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - // std::set properties; - // for (const auto &prop : delta.operation_label_properties.properties) { - // properties.emplace(shard_->NameToProperty(prop)); - // } - // auto ret = shard_->DropUniqueConstraint(shard_->NameToLabel(delta.operation_label_properties.label), - // properties, timestamp); - // if (ret != UniqueConstraints::DeletionStatus::SUCCESS) throw utils::BasicException("Invalid transaction!"); - // break; - // } - // } - } - - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid data!"); - - shard_->last_commit_timestamp_ = max_commit_timestamp; - - return applied_deltas; -} -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/replication/replication_server.hpp b/src/storage/v3/replication/replication_server.hpp deleted file mode 100644 index 6b278a103..000000000 --- a/src/storage/v3/replication/replication_server.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include "storage/v3/shard.hpp" - -namespace memgraph::storage::v3 { - -class Shard::ReplicationServer { - public: - explicit ReplicationServer(Shard *shard, io::network::Endpoint endpoint, - const replication::ReplicationServerConfig &config); - ReplicationServer(const ReplicationServer &) = delete; - ReplicationServer(ReplicationServer &&) = delete; - ReplicationServer &operator=(const ReplicationServer &) = delete; - ReplicationServer &operator=(ReplicationServer &&) = delete; - - ~ReplicationServer(); - - private: - // RPC handlers - void HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder); - static void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder); - void AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder); - void SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder); - void WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder); - void CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder); - - void LoadWal(replication::Decoder *decoder); - uint64_t ReadAndApplyDelta(durability::BaseDecoder *decoder); - - std::optional rpc_server_context_; - std::optional rpc_server_; - - Shard *shard_; -}; - -} // namespace memgraph::storage::v3 diff --git a/src/storage/v3/replication/rpc.lcp b/src/storage/v3/replication/rpc.lcp deleted file mode 100644 index 55354b392..000000000 --- a/src/storage/v3/replication/rpc.lcp +++ /dev/null @@ -1,74 +0,0 @@ -;; Copyright 2022 Memgraph Ltd. -;; -;; Use of this software is governed by the Business Source License -;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -;; License, and you may not use this file except in compliance with the Business Source License. -;; -;; As of the Change Date specified in that file, in accordance with -;; the Business Source License, use of this software will be governed -;; by the Apache License, Version 2.0, included in the file -;; licenses/APL.txt. - -#>cpp -#pragma once - -#include -#include -#include - -#include "rpc/messages.hpp" -#include "slk/serialization.hpp" -#include "slk/streams.hpp" -cpp<# -(lcp:namespace memgraph) -(lcp:namespace storage) -(lcp:namespace v3) -(lcp:namespace replication) - -(lcp:define-rpc append-deltas - ;; The actual deltas are sent as additional data using the RPC client's - ;; streaming API for additional data. - (:request - ((previous-commit-timestamp :uint64_t) - (seq-num :uint64_t))) - (:response - ((success :bool) - (current-commit-timestamp :uint64_t)))) - -(lcp:define-rpc heartbeat - (:request - ((main-commit-timestamp :uint64_t) - (epoch-id "std::string"))) - (:response - ((success :bool) - (current-commit-timestamp :uint64_t) - (epoch-id "std::string")))) - -;; FrequentHearthbeat is required because calling Heartbeat takes the storage lock. -;; Configured by `replication_replica_check_delay`. -(lcp:define-rpc frequent-heartbeat - (:request ()) - (:response ((success :bool)))) - -(lcp:define-rpc snapshot - (:request ()) - (:response - ((success :bool) - (current-commit-timestamp :uint64_t)))) - -(lcp:define-rpc wal-files - (:request ((file-number :uint64_t))) - (:response - ((success :bool) - (current-commit-timestamp :uint64_t)))) - -(lcp:define-rpc current-wal - (:request ()) - (:response - ((success :bool) - (current-commit-timestamp :uint64_t)))) - -(lcp:pop-namespace) ;; replication -(lcp:pop-namespace) ;; v3 -(lcp:pop-namespace) ;; storage -(lcp:pop-namespace) ;; memgraph diff --git a/src/storage/v3/replication/serialization.cpp b/src/storage/v3/replication/serialization.cpp deleted file mode 100644 index ebfabb43f..000000000 --- a/src/storage/v3/replication/serialization.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/replication/serialization.hpp" - -namespace memgraph::storage::v3::replication { -////// Encoder ////// -void Encoder::WriteMarker(durability::Marker marker) { slk::Save(marker, builder_); } - -void Encoder::WriteBool(bool value) { - WriteMarker(durability::Marker::TYPE_BOOL); - slk::Save(value, builder_); -} - -void Encoder::WriteUint(uint64_t value) { - WriteMarker(durability::Marker::TYPE_INT); - slk::Save(value, builder_); -} - -void Encoder::WriteDouble(double value) { - WriteMarker(durability::Marker::TYPE_DOUBLE); - slk::Save(value, builder_); -} - -void Encoder::WriteString(const std::string_view &value) { - WriteMarker(durability::Marker::TYPE_STRING); - slk::Save(value, builder_); -} - -void Encoder::WritePropertyValue(const PropertyValue &value) { - WriteMarker(durability::Marker::TYPE_PROPERTY_VALUE); - slk::Save(value, builder_); -} - -void Encoder::WriteBuffer(const uint8_t *buffer, const size_t buffer_size) { builder_->Save(buffer, buffer_size); } - -void Encoder::WriteFileData(utils::InputFile *file) { - auto file_size = file->GetSize(); - uint8_t buffer[utils::kFileBufferSize]; - while (file_size > 0) { - const auto chunk_size = std::min(file_size, utils::kFileBufferSize); - file->Read(buffer, chunk_size); - WriteBuffer(buffer, chunk_size); - file_size -= chunk_size; - } -} - -void Encoder::WriteFile(const std::filesystem::path &path) { - utils::InputFile file; - MG_ASSERT(file.Open(path), "Failed to open file {}", path); - MG_ASSERT(path.has_filename(), "Path does not have a filename!"); - const auto &filename = path.filename().generic_string(); - WriteString(filename); - auto file_size = file.GetSize(); - WriteUint(file_size); - WriteFileData(&file); - file.Close(); -} - -////// Decoder ////// -std::optional Decoder::ReadMarker() { - durability::Marker marker{durability::Marker::TYPE_NULL}; - slk::Load(&marker, reader_); - return marker; -} - -std::optional Decoder::ReadBool() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_BOOL) return std::nullopt; - bool value{false}; - slk::Load(&value, reader_); - return value; -} - -std::optional Decoder::ReadUint() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_INT) return std::nullopt; - uint64_t value{0}; - slk::Load(&value, reader_); - return value; -} - -std::optional Decoder::ReadDouble() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_DOUBLE) return std::nullopt; - double value{0.0}; - slk::Load(&value, reader_); - return value; -} - -std::optional Decoder::ReadString() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_STRING) return std::nullopt; - std::string value; - slk::Load(&value, reader_); - return std::move(value); -} - -std::optional Decoder::ReadPropertyValue() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_PROPERTY_VALUE) - return std::nullopt; - PropertyValue value; - slk::Load(&value, reader_); - return std::move(value); -} - -bool Decoder::SkipString() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_STRING) return false; - std::string value; - slk::Load(&value, reader_); - return true; -} - -bool Decoder::SkipPropertyValue() { - if (const auto marker = ReadMarker(); !marker || marker != durability::Marker::TYPE_PROPERTY_VALUE) return false; - PropertyValue value; - slk::Load(&value, reader_); - return true; -} - -std::optional Decoder::ReadFile(const std::filesystem::path &directory, - const std::string &suffix) { - MG_ASSERT(std::filesystem::exists(directory) && std::filesystem::is_directory(directory), - "Sent path for streamed files should be a valid directory!"); - utils::OutputFile file; - const auto maybe_filename = ReadString(); - MG_ASSERT(maybe_filename, "Filename missing for the file"); - const auto filename = *maybe_filename + suffix; - auto path = directory / filename; - - file.Open(path, utils::OutputFile::Mode::OVERWRITE_EXISTING); - std::optional maybe_file_size = ReadUint(); - MG_ASSERT(maybe_file_size, "File size missing"); - auto file_size = *maybe_file_size; - uint8_t buffer[utils::kFileBufferSize]; - while (file_size > 0) { - const auto chunk_size = std::min(file_size, utils::kFileBufferSize); - reader_->Load(buffer, chunk_size); - file.Write(buffer, chunk_size); - file_size -= chunk_size; - } - file.Close(); - return std::move(path); -} -} // namespace memgraph::storage::v3::replication diff --git a/src/storage/v3/replication/serialization.hpp b/src/storage/v3/replication/serialization.hpp deleted file mode 100644 index d769058ca..000000000 --- a/src/storage/v3/replication/serialization.hpp +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include - -#include "slk/streams.hpp" -#include "storage/v3/durability/serialization.hpp" -#include "storage/v3/replication/slk.hpp" -#include "utils/cast.hpp" -#include "utils/file.hpp" - -namespace memgraph::storage::v3::replication { - -class Encoder final : public durability::BaseEncoder { - public: - explicit Encoder(slk::Builder *builder) : builder_(builder) {} - - void WriteMarker(durability::Marker marker) override; - - void WriteBool(bool value) override; - - void WriteUint(uint64_t value) override; - - void WriteDouble(double value) override; - - void WriteString(const std::string_view &value) override; - - void WritePropertyValue(const PropertyValue &value) override; - - void WriteBuffer(const uint8_t *buffer, size_t buffer_size); - - void WriteFileData(utils::InputFile *file); - - void WriteFile(const std::filesystem::path &path); - - private: - slk::Builder *builder_; -}; - -class Decoder final : public durability::BaseDecoder { - public: - explicit Decoder(slk::Reader *reader) : reader_(reader) {} - - std::optional ReadMarker() override; - - std::optional ReadBool() override; - - std::optional ReadUint() override; - - std::optional ReadDouble() override; - - std::optional ReadString() override; - - std::optional ReadPropertyValue() override; - - bool SkipString() override; - - bool SkipPropertyValue() override; - - /// Read the file and save it inside the specified directory. - /// @param directory Directory which will contain the read file. - /// @param suffix Suffix to be added to the received file's filename. - /// @return If the read was successful, path to the read file. - std::optional ReadFile(const std::filesystem::path &directory, const std::string &suffix = ""); - - private: - slk::Reader *reader_; -}; - -} // namespace memgraph::storage::v3::replication diff --git a/src/storage/v3/replication/slk.cpp b/src/storage/v3/replication/slk.cpp deleted file mode 100644 index 5117fe6e8..000000000 --- a/src/storage/v3/replication/slk.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v3/replication/slk.hpp" - -#include - -#include "storage/v3/property_value.hpp" -#include "storage/v3/temporal.hpp" -#include "utils/cast.hpp" - -namespace memgraph::slk { - -void Save(const storage::v3::Gid &gid, slk::Builder *builder) { slk::Save(gid.AsUint(), builder); } - -void Load(storage::v3::Gid *gid, slk::Reader *reader) { - uint64_t value{0}; - slk::Load(&value, reader); - *gid = storage::v3::Gid::FromUint(value); -} - -void Load(storage::v3::PropertyValue::Type *type, slk::Reader *reader) { - using PVTypeUnderlyingType = std::underlying_type_t; - PVTypeUnderlyingType value{}; - slk::Load(&value, reader); - bool valid{false}; - switch (value) { - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::Null): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::Bool): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::Int): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::Double): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::String): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::List): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::Map): - case utils::UnderlyingCast(storage::v3::PropertyValue::Type::TemporalData): - valid = true; - break; - default: - valid = false; - break; - } - if (!valid) throw slk::SlkDecodeException("Trying to load unknown storage::v3::PropertyValue!"); - *type = static_cast(value); -} - -void Save(const storage::v3::PropertyValue &value, slk::Builder *builder) { - switch (value.type()) { - case storage::v3::PropertyValue::Type::Null: - slk::Save(storage::v3::PropertyValue::Type::Null, builder); - return; - case storage::v3::PropertyValue::Type::Bool: - slk::Save(storage::v3::PropertyValue::Type::Bool, builder); - slk::Save(value.ValueBool(), builder); - return; - case storage::v3::PropertyValue::Type::Int: - slk::Save(storage::v3::PropertyValue::Type::Int, builder); - slk::Save(value.ValueInt(), builder); - return; - case storage::v3::PropertyValue::Type::Double: - slk::Save(storage::v3::PropertyValue::Type::Double, builder); - slk::Save(value.ValueDouble(), builder); - return; - case storage::v3::PropertyValue::Type::String: - slk::Save(storage::v3::PropertyValue::Type::String, builder); - slk::Save(value.ValueString(), builder); - return; - case storage::v3::PropertyValue::Type::List: { - slk::Save(storage::v3::PropertyValue::Type::List, builder); - const auto &values = value.ValueList(); - size_t size = values.size(); - slk::Save(size, builder); - for (const auto &v : values) { - slk::Save(v, builder); - } - return; - } - case storage::v3::PropertyValue::Type::Map: { - slk::Save(storage::v3::PropertyValue::Type::Map, builder); - const auto &map = value.ValueMap(); - size_t size = map.size(); - slk::Save(size, builder); - for (const auto &kv : map) { - slk::Save(kv, builder); - } - return; - } - case storage::v3::PropertyValue::Type::TemporalData: { - slk::Save(storage::v3::PropertyValue::Type::TemporalData, builder); - const auto temporal_data = value.ValueTemporalData(); - slk::Save(temporal_data.type, builder); - slk::Save(temporal_data.microseconds, builder); - return; - } - } -} - -void Load(storage::v3::PropertyValue *value, slk::Reader *reader) { - storage::v3::PropertyValue::Type type{}; - slk::Load(&type, reader); - switch (type) { - case storage::v3::PropertyValue::Type::Null: - *value = storage::v3::PropertyValue(); - return; - case storage::v3::PropertyValue::Type::Bool: { - bool v{false}; - slk::Load(&v, reader); - *value = storage::v3::PropertyValue(v); - return; - } - case storage::v3::PropertyValue::Type::Int: { - int64_t v{0}; - slk::Load(&v, reader); - *value = storage::v3::PropertyValue(v); - return; - } - case storage::v3::PropertyValue::Type::Double: { - double v{0.0}; - slk::Load(&v, reader); - *value = storage::v3::PropertyValue(v); - return; - } - case storage::v3::PropertyValue::Type::String: { - std::string v; - slk::Load(&v, reader); - *value = storage::v3::PropertyValue(std::move(v)); - return; - } - case storage::v3::PropertyValue::Type::List: { - size_t size{0}; - slk::Load(&size, reader); - std::vector list(size); - for (size_t i = 0; i < size; ++i) { - slk::Load(&list[i], reader); - } - *value = storage::v3::PropertyValue(std::move(list)); - return; - } - case storage::v3::PropertyValue::Type::Map: { - size_t size{0}; - slk::Load(&size, reader); - std::map map; - for (size_t i = 0; i < size; ++i) { - std::pair kv; - slk::Load(&kv, reader); - map.insert(kv); - } - *value = storage::v3::PropertyValue(std::move(map)); - return; - } - case storage::v3::PropertyValue::Type::TemporalData: { - storage::v3::TemporalType temporal_type{}; - slk::Load(&temporal_type, reader); - int64_t microseconds{0}; - slk::Load(µseconds, reader); - *value = storage::v3::PropertyValue(storage::v3::TemporalData{temporal_type, microseconds}); - return; - } - } -} - -} // namespace memgraph::slk diff --git a/src/storage/v3/replication/slk.hpp b/src/storage/v3/replication/slk.hpp deleted file mode 100644 index 5464a4aff..000000000 --- a/src/storage/v3/replication/slk.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2022 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include "slk/serialization.hpp" -#include "storage/v3/durability/marker.hpp" -#include "storage/v3/id_types.hpp" -#include "storage/v3/property_value.hpp" -#include "utils/concepts.hpp" - -namespace memgraph::slk { - -void Save(const storage::v3::Gid &gid, slk::Builder *builder); -void Load(storage::v3::Gid *gid, slk::Reader *reader); - -void Save(const storage::v3::PropertyValue &value, slk::Builder *builder); -void Load(storage::v3::PropertyValue *value, slk::Reader *reader); - -template -void Save(const T &enum_value, slk::Builder *builder) { - slk::Save(utils::UnderlyingCast(enum_value), builder); -} - -template -void Load(T *enum_value, slk::Reader *reader) { - using UnderlyingType = std::underlying_type_t; - UnderlyingType value; - slk::Load(&value, reader); - *enum_value = static_cast(value); -} - -} // namespace memgraph::slk diff --git a/src/storage/v3/shard.cpp b/src/storage/v3/shard.cpp index ae9a224d5..94aff2b62 100644 --- a/src/storage/v3/shard.cpp +++ b/src/storage/v3/shard.cpp @@ -12,7 +12,6 @@ #include "storage/v3/shard.hpp" #include -#include #include #include #include @@ -25,22 +24,13 @@ #include #include "io/network/endpoint.hpp" -#include "storage/v3/constraints.hpp" -#include "storage/v3/durability/durability.hpp" -#include "storage/v3/durability/metadata.hpp" -#include "storage/v3/durability/paths.hpp" -#include "storage/v3/durability/snapshot.hpp" -#include "storage/v3/durability/wal.hpp" +#include "io/time.hpp" #include "storage/v3/edge_accessor.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/indices.hpp" #include "storage/v3/key_store.hpp" #include "storage/v3/mvcc.hpp" #include "storage/v3/property_value.hpp" -#include "storage/v3/replication/config.hpp" -#include "storage/v3/replication/replication_client.hpp" -#include "storage/v3/replication/replication_server.hpp" -#include "storage/v3/replication/rpc.hpp" #include "storage/v3/schema_validator.hpp" #include "storage/v3/transaction.hpp" #include "storage/v3/vertex.hpp" @@ -63,16 +53,30 @@ namespace memgraph::storage::v3 { using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; namespace { -inline constexpr uint16_t kEpochHistoryRetention = 1000; +uint64_t GetCleanupBeforeTimestamp(const std::map> &transactions, + const io::Time clean_up_before) { + MG_ASSERT(!transactions.empty(), "There are no transactions!"); + const auto it = std::lower_bound( + transactions.begin(), transactions.end(), clean_up_before, + // A different comparator has to be used here as this function translates the wall clock time into a logical id + // NOLINTNEXTLINE(performance-inefficient-algorithm) + [](const std::pair> &trans, const io::Time clean_up_before) { + return trans.second->start_timestamp.coordinator_wall_clock < clean_up_before; + }); + if (it == transactions.end()) { + // all of the transaction are old enough to be cleaned up, return a timestamp that is higher than all of them + return transactions.rbegin()->first + 1; + } + return it->first; +} -void InsertVertexPKIntoList(auto &container, const PrimaryKey &primary_key) { container.push_back(primary_key); } } // namespace auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Iterator end, std::optional *vertex, Transaction *tx, View view, Indices *indices, - Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator) { + Config::Items config, const VertexValidator &vertex_validator) { while (it != end) { - *vertex = VertexAccessor::Create(&it->vertex, tx, indices, constraints, config, vertex_validator, view); + *vertex = VertexAccessor::Create(&it->vertex, tx, indices, config, vertex_validator, view); if (!*vertex) { ++it; continue; @@ -85,14 +89,14 @@ auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Ite AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it) : self_(self), it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), &self->vertex_, self->transaction_, self->view_, - self->indices_, self_->constraints_, self->config_, *self_->vertex_validator_)) {} + self->indices_, self->config_, *self_->vertex_validator_)) {} VertexAccessor AllVerticesIterable::Iterator::operator*() const { return *self_->vertex_; } AllVerticesIterable::Iterator &AllVerticesIterable::Iterator::operator++() { ++it_; it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_.end(), &self_->vertex_, self_->transaction_, self_->view_, - self_->indices_, self_->constraints_, self_->config_, *self_->vertex_validator_); + self_->indices_, self_->config_, *self_->vertex_validator_); return *this; } @@ -322,157 +326,17 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key, max_primary_key_{max_primary_key}, schema_validator_{schemas_}, vertex_validator_{schema_validator_, primary_label}, - indices_{&constraints_, config.items, vertex_validator_}, + indices_{config.items, vertex_validator_}, isolation_level_{config.transaction.isolation_level}, config_{config}, - snapshot_directory_{config_.durability.storage_directory / durability::kSnapshotDirectory}, - wal_directory_{config_.durability.storage_directory / durability::kWalDirectory}, - lock_file_path_{config_.durability.storage_directory / durability::kLockFile}, uuid_{utils::GenerateUUID()}, epoch_id_{utils::GenerateUUID()}, - global_locker_{file_retainer_.AddLocker()} { - if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED && - replication_role_ == ReplicationRole::MAIN) { - spdlog::warn( - "The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider " - "enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because " - "without write-ahead logs this instance is not replicating any data."); - } - if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || - config_.durability.snapshot_on_exit || config_.durability.recover_on_startup) { - // Create the directory initially to crash the database in case of - // permission errors. This is done early to crash the database on startup - // instead of crashing the database for the first time during runtime (which - // could be an unpleasant surprise). - utils::EnsureDirOrDie(snapshot_directory_); - // Same reasoning as above. - utils::EnsureDirOrDie(wal_directory_); + global_locker_{file_retainer_.AddLocker()} {} - // Verify that the user that started the process is the same user that is - // the owner of the storage directory. - durability::VerifyStorageDirectoryOwnerAndProcessUserOrDie(config_.durability.storage_directory); +Shard::~Shard() {} - // Create the lock file and open a handle to it. This will crash the - // database if it can't open the file for writing or if any other process is - // holding the file opened. - lock_file_handle_.Open(lock_file_path_, utils::OutputFile::Mode::OVERWRITE_EXISTING); - MG_ASSERT(lock_file_handle_.AcquireLock(), - "Couldn't acquire lock on the storage directory {}!\n" - "Another Memgraph process is currently running with the same " - "storage directory, please stop it first before starting this " - "process!", - config_.durability.storage_directory); - } - if (config_.durability.recover_on_startup) { - auto info = std::optional{}; - - // durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch_id_, &epoch_history_, &vertices_, - // &edges_, &edge_count_, &name_id_mapper_, &indices_, &constraints_, config_.items, - // &wal_seq_num_); - if (info) { - timestamp_ = std::max(timestamp_, info->next_timestamp); - if (info->last_commit_timestamp) { - last_commit_timestamp_ = *info->last_commit_timestamp; - } - } - } else if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || - config_.durability.snapshot_on_exit) { - bool files_moved = false; - auto backup_root = config_.durability.storage_directory / durability::kBackupDirectory; - for (const auto &[path, dirname, what] : - {std::make_tuple(snapshot_directory_, durability::kSnapshotDirectory, "snapshot"), - std::make_tuple(wal_directory_, durability::kWalDirectory, "WAL")}) { - if (!utils::DirExists(path)) continue; - auto backup_curr = backup_root / dirname; - std::error_code error_code; - for (const auto &item : std::filesystem::directory_iterator(path, error_code)) { - utils::EnsureDirOrDie(backup_root); - utils::EnsureDirOrDie(backup_curr); - std::error_code item_error_code; - std::filesystem::rename(item.path(), backup_curr / item.path().filename(), item_error_code); - MG_ASSERT(!item_error_code, "Couldn't move {} file {} because of: {}", what, item.path(), - item_error_code.message()); - files_moved = true; - } - MG_ASSERT(!error_code, "Couldn't backup {} files because of: {}", what, error_code.message()); - } - if (files_moved) { - spdlog::warn( - "Since Memgraph was not supposed to recover on startup and " - "durability is enabled, your current durability files will likely " - "be overridden. To prevent important data loss, Memgraph has stored " - "those files into a .backup directory inside the storage directory."); - } - } - if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED) { - // TODO(antaljanosbenjamin): handle snapshots - // snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval, [this] { - // if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) { - // switch (maybe_error.GetError()) { - // case CreateSnapshotError::DisabledForReplica: - // spdlog::warn( - // utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication")); - // break; - // } - // } - // }); - } - - if (timestamp_ == kTimestampInitialId) { - commit_log_.emplace(); - } else { - commit_log_.emplace(timestamp_); - } -} - -Shard::~Shard() { - { - // Clear replication data - replication_server_.reset(); - replication_clients_.WithLock([&](auto &clients) { clients.clear(); }); - } - if (wal_file_) { - wal_file_->FinalizeWal(); - wal_file_ = std::nullopt; - } - if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED) { - // TODO(antaljanosbenjamin): stop snapshot creation - } - if (config_.durability.snapshot_on_exit) { - if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) { - switch (maybe_error.GetError()) { - case CreateSnapshotError::DisabledForReplica: - spdlog::warn(utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication")); - break; - } - } - } -} - -Shard::Accessor::Accessor(Shard *shard, IsolationLevel isolation_level) - : shard_(shard), - transaction_(shard->CreateTransaction(isolation_level)), - is_transaction_active_(true), - config_(shard->config_.items) {} - -Shard::Accessor::Accessor(Accessor &&other) noexcept - : shard_(other.shard_), - transaction_(std::move(other.transaction_)), - commit_timestamp_(other.commit_timestamp_), - is_transaction_active_(other.is_transaction_active_), - config_(other.config_) { - // Don't allow the other accessor to abort our transaction in destructor. - other.is_transaction_active_ = false; - other.commit_timestamp_.reset(); -} - -Shard::Accessor::~Accessor() { - if (is_transaction_active_) { - Abort(); - } - - FinalizeTransaction(); -} +Shard::Accessor::Accessor(Shard &shard, Transaction &transaction) + : shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {} ResultSchema Shard::Accessor::CreateVertexAndValidate( LabelId primary_label, const std::vector &labels, @@ -495,11 +359,11 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( })->second); } auto acc = shard_->vertices_.access(); - auto *delta = CreateDeleteObjectDelta(&transaction_); + auto *delta = CreateDeleteObjectDelta(transaction_); auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}}); + delta->prev.Set(&it->vertex); - VertexAccessor vertex_acc{&it->vertex, &transaction_, &shard_->indices_, - &shard_->constraints_, config_, shard_->vertex_validator_}; + VertexAccessor vertex_acc{&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_}; MG_ASSERT(inserted, "The vertex must be inserted here!"); MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!"); @@ -517,7 +381,6 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( return {err.GetError()}; } } - delta->prev.Set(&it->vertex); return vertex_acc; } @@ -533,11 +396,11 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( } OOMExceptionEnabler oom_exception; auto acc = shard_->vertices_.access(); - auto *delta = CreateDeleteObjectDelta(&transaction_); + auto *delta = CreateDeleteObjectDelta(transaction_); auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}}); + delta->prev.Set(&it->vertex); - VertexAccessor vertex_acc{&it->vertex, &transaction_, &shard_->indices_, - &shard_->constraints_, config_, shard_->vertex_validator_}; + VertexAccessor vertex_acc{&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_}; MG_ASSERT(inserted, "The vertex must be inserted here!"); MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!"); @@ -555,7 +418,6 @@ ResultSchema Shard::Accessor::CreateVertexAndValidate( return {err.GetError()}; } } - delta->prev.Set(&it->vertex); return vertex_acc; } @@ -566,17 +428,16 @@ std::optional Shard::Accessor::FindVertex(std::vectorvertex, &transaction_, &shard_->indices_, &shard_->constraints_, config_, - shard_->vertex_validator_, view); + return VertexAccessor::Create(&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view); } Result> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) { - MG_ASSERT(vertex->transaction_ == &transaction_, + MG_ASSERT(vertex->transaction_ == transaction_, "VertexAccessor must be from the same transaction as the storage " "accessor when deleting a vertex!"); auto *vertex_ptr = vertex->vertex_; - if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; + if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; if (vertex_ptr->deleted) { return std::optional{}; @@ -584,18 +445,18 @@ Result> Shard::Accessor::DeleteVertex(VertexAccess if (!vertex_ptr->in_edges.empty() || !vertex_ptr->out_edges.empty()) return Error::VERTEX_HAS_EDGES; - CreateAndLinkDelta(&transaction_, vertex_ptr, Delta::RecreateObjectTag()); + CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag()); vertex_ptr->deleted = true; - return std::make_optional(vertex_ptr, &transaction_, &shard_->indices_, &shard_->constraints_, - config_, shard_->vertex_validator_, true); + return std::make_optional(vertex_ptr, transaction_, &shard_->indices_, config_, + shard_->vertex_validator_, true); } Result>>> Shard::Accessor::DetachDeleteVertex( VertexAccessor *vertex) { using ReturnType = std::pair>; - MG_ASSERT(vertex->transaction_ == &transaction_, + MG_ASSERT(vertex->transaction_ == transaction_, "VertexAccessor must be from the same transaction as the storage " "accessor when deleting a vertex!"); auto *vertex_ptr = vertex->vertex_; @@ -604,7 +465,7 @@ Result>>> Shar std::vector out_edges; { - if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; + if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; if (vertex_ptr->deleted) return std::optional{}; @@ -616,8 +477,7 @@ Result>>> Shar const VertexId vertex_id{shard_->primary_label_, vertex_ptr->keys.Keys()}; for (const auto &item : in_edges) { auto [edge_type, from_vertex, edge] = item; - EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, &transaction_, &shard_->indices_, &shard_->constraints_, - config_); + EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_); auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid()); if (ret.HasError()) { MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!"); @@ -630,8 +490,7 @@ Result>>> Shar } for (const auto &item : out_edges) { auto [edge_type, to_vertex, edge] = item; - EdgeAccessor e(edge, edge_type, vertex_id, to_vertex, &transaction_, &shard_->indices_, &shard_->constraints_, - config_); + EdgeAccessor e(edge, edge_type, vertex_id, to_vertex, transaction_, &shard_->indices_, config_); auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid()); if (ret.HasError()) { MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!"); @@ -647,16 +506,16 @@ Result>>> Shar // vertex. Some other transaction could have modified the vertex in the // meantime if we didn't have any edges to delete. - if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; + if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; MG_ASSERT(!vertex_ptr->deleted, "Invalid database state!"); - CreateAndLinkDelta(&transaction_, vertex_ptr, Delta::RecreateObjectTag()); + CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag()); vertex_ptr->deleted = true; - return std::make_optional(VertexAccessor{vertex_ptr, &transaction_, &shard_->indices_, - &shard_->constraints_, config_, shard_->vertex_validator_, true}, - std::move(deleted_edges)); + return std::make_optional( + VertexAccessor{vertex_ptr, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, true}, + std::move(deleted_edges)); } Result Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, @@ -684,18 +543,18 @@ Result Shard::Accessor::CreateEdge(VertexId from_vertex_id, Vertex } if (from_is_local) { - if (!PrepareForWrite(&transaction_, from_vertex)) return Error::SERIALIZATION_ERROR; + if (!PrepareForWrite(transaction_, from_vertex)) return Error::SERIALIZATION_ERROR; if (from_vertex->deleted) return Error::DELETED_OBJECT; } if (to_is_local && to_vertex != from_vertex) { - if (!PrepareForWrite(&transaction_, to_vertex)) return Error::SERIALIZATION_ERROR; + if (!PrepareForWrite(transaction_, to_vertex)) return Error::SERIALIZATION_ERROR; if (to_vertex->deleted) return Error::DELETED_OBJECT; } EdgeRef edge(gid); if (config_.properties_on_edges) { auto acc = shard_->edges_.access(); - auto *delta = CreateDeleteObjectDelta(&transaction_); + auto *delta = CreateDeleteObjectDelta(transaction_); auto [it, inserted] = acc.insert(Edge(gid, delta)); MG_ASSERT(inserted, "The edge must be inserted here!"); MG_ASSERT(it != acc.end(), "Invalid Edge accessor!"); @@ -704,18 +563,18 @@ Result Shard::Accessor::CreateEdge(VertexId from_vertex_id, Vertex } if (from_is_local) { - CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex_id, edge); + CreateAndLinkDelta(transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex_id, edge); from_vertex->out_edges.emplace_back(edge_type, to_vertex_id, edge); } if (to_is_local) { - CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex_id, edge); + CreateAndLinkDelta(transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex_id, edge); to_vertex->in_edges.emplace_back(edge_type, from_vertex_id, edge); } // Increment edge count. ++shard_->edge_count_; - return EdgeAccessor(edge, edge_type, std::move(from_vertex_id), std::move(to_vertex_id), &transaction_, - &shard_->indices_, &shard_->constraints_, config_); + return EdgeAccessor(edge, edge_type, std::move(from_vertex_id), std::move(to_vertex_id), transaction_, + &shard_->indices_, config_); } Result> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, @@ -743,13 +602,13 @@ Result> Shard::Accessor::DeleteEdge(VertexId from_ve MG_ASSERT(from_is_local || to_is_local, "Trying to delete an edge without having a local vertex"); if (from_is_local) { - if (!PrepareForWrite(&transaction_, from_vertex)) { + if (!PrepareForWrite(transaction_, from_vertex)) { return Error::SERIALIZATION_ERROR; } MG_ASSERT(!from_vertex->deleted, "Invalid database state!"); } if (to_is_local && to_vertex != from_vertex) { - if (!PrepareForWrite(&transaction_, to_vertex)) { + if (!PrepareForWrite(transaction_, to_vertex)) { return Error::SERIALIZATION_ERROR; } MG_ASSERT(!to_vertex->deleted, "Invalid database state!"); @@ -800,24 +659,24 @@ Result> Shard::Accessor::DeleteEdge(VertexId from_ve if (config_.properties_on_edges) { auto *edge_ptr = edge_ref.ptr; - CreateAndLinkDelta(&transaction_, edge_ptr, Delta::RecreateObjectTag()); + CreateAndLinkDelta(transaction_, edge_ptr, Delta::RecreateObjectTag()); edge_ptr->deleted = true; } MG_ASSERT(edge_type.has_value(), "Edge type is not determined"); if (from_is_local) { - CreateAndLinkDelta(&transaction_, from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref); + CreateAndLinkDelta(transaction_, from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref); } if (to_is_local) { - CreateAndLinkDelta(&transaction_, to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref); + CreateAndLinkDelta(transaction_, to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref); } // Decrement edge count. --shard_->edge_count_; return std::make_optional(edge_ref, *edge_type, std::move(from_vertex_id), std::move(to_vertex_id), - &transaction_, &shard_->indices_, &shard_->constraints_, config_, true); + transaction_, &shard_->indices_, config_, true); } LabelId Shard::Accessor::NameToLabel(std::string_view name) const { return shard_->NameToLabel(name); } @@ -836,126 +695,43 @@ const std::string &Shard::Accessor::EdgeTypeToName(EdgeTypeId edge_type) const { return shard_->EdgeTypeToName(edge_type); } -void Shard::Accessor::AdvanceCommand() { ++transaction_.command_id; } +void Shard::Accessor::AdvanceCommand() { ++transaction_->command_id; } -utils::BasicResult Shard::Accessor::Commit( - const std::optional desired_commit_timestamp) { - MG_ASSERT(is_transaction_active_, "The transaction is already terminated!"); - MG_ASSERT(!transaction_.must_abort, "The transaction can't be committed!"); - - if (transaction_.deltas.empty()) { - // We don't have to update the commit timestamp here because no one reads - // it. - shard_->commit_log_->MarkFinished(transaction_.start_timestamp); - } else { - // Validate that existence constraints are satisfied for all modified - // vertices. - for (const auto &delta : transaction_.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) { - continue; - } - // No need to take any locks here because we modified this vertex and no - // one else can touch it until we commit. - auto validation_result = ValidateExistenceConstraints(*prev.vertex, shard_->constraints_); - if (validation_result) { - Abort(); - return {*validation_result}; - } - } - - // Result of validating the vertex against unique constraints. It has to be - // declared outside of the critical section scope because its value is - // tested for Abort call which has to be done out of the scope. - std::optional unique_constraint_violation; - - // Save these so we can mark them used in the commit log. - uint64_t start_timestamp = transaction_.start_timestamp; - - commit_timestamp_.emplace(shard_->CommitTimestamp(desired_commit_timestamp)); - - // Before committing and validating vertices against unique constraints, - // we have to update unique constraints with the vertices that are going - // to be validated/committed. - for (const auto &delta : transaction_.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) { - continue; - } - shard_->constraints_.unique_constraints.UpdateBeforeCommit(prev.vertex, transaction_); - } - - // Validate that unique constraints are satisfied for all modified - // vertices. - for (const auto &delta : transaction_.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) { - continue; - } - - // No need to take any locks here because we modified this vertex and no - // one else can touch it until we commit. - unique_constraint_violation = - shard_->constraints_.unique_constraints.Validate(*prev.vertex, transaction_, *commit_timestamp_); - if (unique_constraint_violation) { - break; - } - } - - if (!unique_constraint_violation) { - // Write transaction to WAL while holding the engine lock to make sure - // that committed transactions are sorted by the commit timestamp in the - // WAL files. We supply the new commit timestamp to the function so that - // it knows what will be the final commit timestamp. The WAL must be - // written before actually committing the transaction (before setting - // the commit timestamp) so that no other transaction can see the - // modifications before they are written to disk. - // Replica can log only the write transaction received from Main - // so the Wal files are consistent - if (shard_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) { - shard_->AppendToWal(transaction_, *commit_timestamp_); - } - - // TODO(antaljanosbenjamin): Figure out: - // 1. How the committed transactions are sorted in `committed_transactions_` - // 2. Why it was necessary to lock `committed_transactions_` when it was not accessed at all - // TODO: Update all deltas to have a local copy of the commit timestamp - MG_ASSERT(transaction_.commit_timestamp != nullptr, "Invalid database state!"); - transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); - // Replica can only update the last commit timestamp with - // the commits received from main. - if (shard_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) { - // Update the last commit timestamp - shard_->last_commit_timestamp_ = *commit_timestamp_; - } - - shard_->commit_log_->MarkFinished(start_timestamp); - } - - if (unique_constraint_violation) { - Abort(); - return {*unique_constraint_violation}; - } - } - is_transaction_active_ = false; - - return {}; +void Shard::Accessor::Commit(coordinator::Hlc commit_timestamp) { + MG_ASSERT(!transaction_->is_aborted, "The transaction is already aborted!"); + MG_ASSERT(!transaction_->must_abort, "The transaction can't be committed!"); + MG_ASSERT(transaction_->start_timestamp.logical_id < commit_timestamp.logical_id, + "Commit timestamp must be older than start timestamp!"); + MG_ASSERT(!transaction_->commit_info->is_locally_committed, "The transaction is already committed!"); + transaction_->commit_info->start_or_commit_timestamp = commit_timestamp; + transaction_->commit_info->is_locally_committed = true; } void Shard::Accessor::Abort() { - MG_ASSERT(is_transaction_active_, "The transaction is already terminated!"); + MG_ASSERT(!transaction_->is_aborted, "The transaction is already aborted!"); + MG_ASSERT(!transaction_->commit_info->is_locally_committed, "The transaction is already committed!"); - for (const auto &delta : transaction_.deltas) { + // Here we walk the delta chain from the beginning for each object. That means we only have to start the walking of + // the delta chain when we find a delta which previous pointer points to a vertex or an edge. Every delta will be + // visited by the walks initiated from such deltas. + // This has to be true in all circumstances, as the current transaction is not committed, therefore no other + // transaction can modify the object if this transaction already modified it, thus we can be sure that for each object + // (vertex or edge) that this transaction modified, the deltas will be in the beginning of the delta chain: + // * If an uncommitted transaction has two or more deltas for an object, then those deltas are always strictly + // following each other without having deltas from other transactions among themself, otherwise there will be a + // serialization error between the two transaction. + // * If there would be another delta in front of the deltas of this transactions, that would mean a serialization + // error as the current transaction is not yet committed, thus the deltas of any uncommitted transactions are always + // the first in the chain. + // * There might be deltas and the end of the delta chain though, but only from transactions that are committed + // before the transaction that is being cleaned up. + for (const auto &delta : transaction_->deltas) { auto prev = delta.prev.Get(); switch (prev.type) { case PreviousPtr::Type::VERTEX: { auto *vertex = prev.vertex; Delta *current = vertex->delta; - while (current != nullptr && - current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) { + while (current != nullptr && current->commit_info->start_or_commit_timestamp == transaction_->start_timestamp) { switch (current->action) { case Delta::Action::REMOVE_LABEL: { auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label); @@ -1020,7 +796,7 @@ void Shard::Accessor::Abort() { } case Delta::Action::DELETE_OBJECT: { vertex->deleted = true; - InsertVertexPKIntoList(shard_->deleted_vertices_, vertex->keys.Keys()); + shard_->deleted_vertices_.push_back(vertex->keys.Keys()); break; } case Delta::Action::RECREATE_OBJECT: { @@ -1028,7 +804,7 @@ void Shard::Accessor::Abort() { break; } } - current = current->next.load(std::memory_order_acquire); + current = current->next; } vertex->delta = current; if (current != nullptr) { @@ -1040,8 +816,7 @@ void Shard::Accessor::Abort() { case PreviousPtr::Type::EDGE: { auto *edge = prev.edge; Delta *current = edge->delta; - while (current != nullptr && - current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) { + while (current != nullptr && current->commit_info->start_or_commit_timestamp == transaction_->start_timestamp) { switch (current->action) { case Delta::Action::SET_PROPERTY: { edge->properties.SetProperty(current->property.key, current->property.value); @@ -1066,7 +841,7 @@ void Shard::Accessor::Abort() { break; } } - current = current->next.load(std::memory_order_acquire); + current = current->next; } edge->delta = current; if (current != nullptr) { @@ -1082,24 +857,10 @@ void Shard::Accessor::Abort() { } } - { - uint64_t mark_timestamp = shard_->timestamp_; + transaction_->deltas.clear(); - // Release engine lock because we don't have to hold it anymore and - // emplace back could take a long time. - shard_->garbage_undo_buffers_.emplace_back(mark_timestamp, std::move(transaction_.deltas)); - } - - shard_->commit_log_->MarkFinished(transaction_.start_timestamp); - is_transaction_active_ = false; -} - -void Shard::Accessor::FinalizeTransaction() { - if (commit_timestamp_) { - shard_->commit_log_->MarkFinished(*commit_timestamp_); - shard_->committed_transactions_.emplace_back(std::move(transaction_)); - commit_timestamp_.reset(); - } + transaction_->is_aborted = true; + shard_->has_any_transaction_aborted_since_last_gc = true; } LabelId Shard::NameToLabel(std::string_view name) const { return LabelId::FromUint(name_id_mapper_.NameToId(name)); } @@ -1122,19 +883,16 @@ const std::string &Shard::EdgeTypeToName(EdgeTypeId edge_type) const { return name_id_mapper_.IdToName(edge_type.AsUint()); } -bool Shard::CreateIndex(LabelId label, const std::optional desired_commit_timestamp) { +bool Shard::CreateIndex(LabelId label, const std::optional /*desired_commit_timestamp*/) { // TODO(jbajic) response should be different when label == primary_label if (label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_.access())) { return false; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::LABEL_INDEX_CREATE, label, {}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; return true; } -bool Shard::CreateIndex(LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { +bool Shard::CreateIndex(LabelId label, PropertyId property, + const std::optional /*desired_commit_timestamp*/) { // TODO(jbajic) response should be different when index conflicts with schema if (label == primary_label_ && schemas_.GetSchema(primary_label_)->second.size() == 1 && schemas_.GetSchema(primary_label_)->second[0].property_id == property) { @@ -1144,94 +902,23 @@ bool Shard::CreateIndex(LabelId label, PropertyId property, const std::optional< if (!indices_.label_property_index.CreateIndex(label, property, vertices_.access())) { return false; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE, label, {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; return true; } -bool Shard::DropIndex(LabelId label, const std::optional desired_commit_timestamp) { - if (!indices_.label_index.DropIndex(label)) return false; - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::LABEL_INDEX_DROP, label, {}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return true; +bool Shard::DropIndex(LabelId label, const std::optional /*desired_commit_timestamp*/) { + return indices_.label_index.DropIndex(label); } -bool Shard::DropIndex(LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - if (!indices_.label_property_index.DropIndex(label, property)) return false; - // For a description why using `timestamp_` is correct, see - // `CreateIndex(LabelId label)`. - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP, label, {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return true; +bool Shard::DropIndex(LabelId label, PropertyId property, const std::optional /*desired_commit_timestamp*/) { + return indices_.label_property_index.DropIndex(label, property); } IndicesInfo Shard::ListAllIndices() const { return {indices_.label_index.ListIndices(), indices_.label_property_index.ListIndices()}; } -utils::BasicResult Shard::CreateExistenceConstraint( - LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - // TODO Fix constraints - // auto ret = ::memgraph::storage::v3::CreateExistenceConstraint(&constraints_, label, property, vertices_.access()); - // if (ret.HasError() || !ret.GetValue()) return ret; - return false; - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE, label, {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return true; -} - -bool Shard::DropExistenceConstraint(LabelId label, PropertyId property, - const std::optional desired_commit_timestamp) { - if (!memgraph::storage::v3::DropExistenceConstraint(&constraints_, label, property)) return false; - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP, label, {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return true; -} - -utils::BasicResult Shard::CreateUniqueConstraint( - LabelId label, const std::set &properties, const std::optional desired_commit_timestamp) { - // TODO Fix constraints - // auto ret = constraints_.unique_constraints.CreateConstraint(label, properties, vertices_.access()); - // if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) { - // return ret; - // } - return UniqueConstraints::CreationStatus::ALREADY_EXISTS; - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE, label, properties, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return UniqueConstraints::CreationStatus::SUCCESS; -} - -UniqueConstraints::DeletionStatus Shard::DropUniqueConstraint(LabelId label, const std::set &properties, - const std::optional desired_commit_timestamp) { - auto ret = constraints_.unique_constraints.DropConstraint(label, properties); - if (ret != UniqueConstraints::DeletionStatus::SUCCESS) { - return ret; - } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - AppendToWal(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP, label, properties, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - last_commit_timestamp_ = commit_timestamp; - return UniqueConstraints::DeletionStatus::SUCCESS; -} - const SchemaValidator &Shard::Accessor::GetSchemaValidator() const { return shard_->schema_validator_; } -ConstraintsInfo Shard::ListAllConstraints() const { - return {ListExistenceConstraints(constraints_), constraints_.unique_constraints.ListConstraints()}; -} - SchemasInfo Shard::ListAllSchemas() const { return {schemas_.ListSchemas()}; } const Schemas::Schema *Shard::GetSchema(const LabelId primary_label) const { return schemas_.GetSchema(primary_label); } @@ -1248,673 +935,166 @@ StorageInfo Shard::GetInfo() const { if (vertex_count) { average_degree = 2.0 * static_cast(edge_count_) / static_cast(vertex_count); } - return {vertex_count, edge_count_, average_degree, utils::GetMemoryUsage(), - utils::GetDirDiskUsage(config_.durability.storage_directory)}; + return {vertex_count, edge_count_, average_degree, utils::GetMemoryUsage()}; } VerticesIterable Shard::Accessor::Vertices(LabelId label, View view) { - return VerticesIterable(shard_->indices_.label_index.Vertices(label, view, &transaction_)); + return VerticesIterable(shard_->indices_.label_index.Vertices(label, view, transaction_)); } VerticesIterable Shard::Accessor::Vertices(LabelId label, PropertyId property, View view) { return VerticesIterable( - shard_->indices_.label_property_index.Vertices(label, property, std::nullopt, std::nullopt, view, &transaction_)); + shard_->indices_.label_property_index.Vertices(label, property, std::nullopt, std::nullopt, view, transaction_)); } VerticesIterable Shard::Accessor::Vertices(LabelId label, PropertyId property, const PropertyValue &value, View view) { return VerticesIterable(shard_->indices_.label_property_index.Vertices( - label, property, utils::MakeBoundInclusive(value), utils::MakeBoundInclusive(value), view, &transaction_)); + label, property, utils::MakeBoundInclusive(value), utils::MakeBoundInclusive(value), view, transaction_)); } VerticesIterable Shard::Accessor::Vertices(LabelId label, PropertyId property, const std::optional> &lower_bound, const std::optional> &upper_bound, View view) { return VerticesIterable( - shard_->indices_.label_property_index.Vertices(label, property, lower_bound, upper_bound, view, &transaction_)); + shard_->indices_.label_property_index.Vertices(label, property, lower_bound, upper_bound, view, transaction_)); } -Transaction Shard::CreateTransaction(IsolationLevel isolation_level) { - uint64_t transaction_id{0}; - uint64_t start_timestamp{0}; - - transaction_id = transaction_id_++; - // Replica should have only read queries and the write queries - // can come from main instance with any past timestamp. - // To preserve snapshot isolation we set the start timestamp - // of any query on replica to the last commited transaction - // which is timestamp_ as only commit of transaction with writes - // can change the value of it. - if (replication_role_ == ReplicationRole::REPLICA) { - start_timestamp = timestamp_; - } else { - start_timestamp = timestamp_++; - } - - return {transaction_id, start_timestamp, isolation_level}; -} - -// `force` means there are no active transactions, so everything can be deleted without worrying about removing some -// data that is used by an active transaction -template -void Shard::CollectGarbage() { - if constexpr (force) { - // TODO(antaljanosbenjamin): figure out whether is there any active transaction or not (probably accessors should - // increment/decrement a counter). If there are no transactions, then garbage collection can be forced - CollectGarbage(); +void Shard::CollectGarbage(const io::Time current_time) { + if (start_logical_id_to_transaction_.empty()) { + // There is no transactions that the shard is aware of, thus no aborted or committed transactions, there is nothing + // to clean up return; } - // Garbage collection must be performed in two phases. In the first phase, - // deltas that won't be applied by any transaction anymore are unlinked from - // the version chains. They cannot be deleted immediately, because there - // might be a transaction that still needs them to terminate the version - // chain traversal. They are instead marked for deletion and will be deleted - // in the second GC phase in this GC iteration or some of the following - // ones. + const auto clean_up_before_wall_clock = current_time - config_.gc.reclamation_interval; - uint64_t oldest_active_start_timestamp = commit_log_->OldestActive(); - // We don't move undo buffers of unlinked transactions to garbage_undo_buffers - // list immediately, because we would have to repeatedly take - // garbage_undo_buffers lock. - std::list>> unlinked_undo_buffers; + /// The clean up must happen based on the start timestamp and not on the commit timestamp, thus we have to make sure + /// that we only compare this timestamp to the start timestamp of the transactions and not to the commit timestamp. + const auto clean_up_before_timestamp = + GetCleanupBeforeTimestamp(start_logical_id_to_transaction_, clean_up_before_wall_clock); - // We will only free vertices deleted up until now in this GC cycle, and we - // will do it after cleaning-up the indices. That way we are sure that all - // vertices that appear in an index also exist in main storage. + auto cleaned_up_committed_transaction = false; - // Flag that will be used to determine whether the Index GC should be run. It - // should be run when there were any items that were cleaned up (there were - // updates between this run of the GC and the previous run of the GC). This - // eliminates high CPU usage when the GC doesn't have to clean up anything. - bool run_index_cleanup = !committed_transactions_.empty() || !garbage_undo_buffers_.empty(); + for (auto it = start_logical_id_to_transaction_.begin(); + it != start_logical_id_to_transaction_.end() && + it->second->start_timestamp.logical_id < clean_up_before_timestamp;) { + auto &transaction = *it->second; - while (true) { - // We don't want to hold the lock on commited transactions for too long, - // because that prevents other transactions from committing. - Transaction *transaction{nullptr}; - { - if (committed_transactions_.empty()) { - break; - } - transaction = &committed_transactions_.front(); - } + if (transaction.commit_info->is_locally_committed) { + cleaned_up_committed_transaction = true; + auto commit_timestamp = transaction.commit_info->start_or_commit_timestamp; - auto commit_timestamp = transaction->commit_timestamp->load(std::memory_order_acquire); - if (commit_timestamp >= oldest_active_start_timestamp) { - break; - } - - // When unlinking a delta which is the first delta in its version chain, - // special care has to be taken to avoid the following race condition: - // - // [Vertex] --> [Delta A] - // - // GC thread: Delta A is the first in its chain, it must be unlinked from - // vertex and marked for deletion - // TX thread: Update vertex and add Delta B with Delta A as next - // - // [Vertex] --> [Delta B] <--> [Delta A] - // - // GC thread: Unlink delta from Vertex - // - // [Vertex] --> (nullptr) - // - // When processing a delta that is the first one in its chain, we - // obtain the corresponding vertex or edge lock, and then verify that this - // delta still is the first in its chain. - // When processing a delta that is in the middle of the chain we only - // process the final delta of the given transaction in that chain. We - // determine the owner of the chain (either a vertex or an edge), obtain the - // corresponding lock, and then verify that this delta is still in the same - // position as it was before taking the lock. - // - // Even though the delta chain is lock-free (both `next` and `prev`) the - // chain should not be modified without taking the lock from the object that - // owns the chain (either a vertex or an edge). Modifying the chain without - // taking the lock will cause subtle race conditions that will leave the - // chain in a broken state. - // The chain can be only read without taking any locks. - - for (Delta &delta : transaction->deltas) { - while (true) { - auto prev = delta.prev.Get(); - switch (prev.type) { - case PreviousPtr::Type::VERTEX: { - Vertex *vertex = prev.vertex; - if (vertex->delta != &delta) { - // Something changed, we're not the first delta in the chain - // anymore. - continue; - } - vertex->delta = nullptr; - if (vertex->deleted) { - InsertVertexPKIntoList(deleted_vertices_, vertex->keys.Keys()); - } - break; - } - case PreviousPtr::Type::EDGE: { - Edge *edge = prev.edge; - if (edge->delta != &delta) { - // Something changed, we're not the first delta in the chain - // anymore. - continue; - } - edge->delta = nullptr; - if (edge->deleted) { - deleted_edges_.push_back(edge->gid); - } - break; - } - case PreviousPtr::Type::DELTA: { - if (prev.delta->timestamp->load(std::memory_order_acquire) == commit_timestamp) { - // The delta that is newer than this one is also a delta from this - // transaction. We skip the current delta and will remove it as a - // part of the suffix later. + // Two important notes here: + // 1. Deltas of the transaction in an object's deltachain are always strictly following each other without having + // deltas from other transactions between them. + // 2. While the deltas are cleaned up, we can be sure the deltas of the actually cleaned up transaction is always + // the last deltas in the delta chain. + // Reasoning: + // 1. Once a delta is added to the delta chain of the object, no other transaction can add a delta to that + // deltachain until the transaction which just installed the delta is aborted or committed. + // 2. The deltas are sorted by start timestamp and cleaned up in that order. This means it is enough to prove that + // in case of any two deltas from two transactions, the one with smaller start timestamp will be later in the + // deltachain (starting from the object) compared to the one with greater start timestamp. Let's assume two + // transactions, t1 and t2 that are started in this order. If t2 modifies an object first, then it will put its + // delta in the beginning of the deltachain. If t1 tries to modify the same object, a serialization error must + // happen regardless whether t2 committed or not, as the first delta in the deltachain has a bigger timestamp than + // the start timestamp of t1, because both the start and commit timestamp of t2 is larger than the start timestamp + // of t1. Thus the only way t1 can modify the same object after t2 modified it, if t2 aborts. In that case the + // deltas of t2 are cleaned up during abort. + for (Delta &delta : transaction.deltas) { + while (true) { + auto prev = delta.prev.Get(); + switch (prev.type) { + case PreviousPtr::Type::VERTEX: { + Vertex *vertex = prev.vertex; + vertex->delta = nullptr; + if (vertex->deleted) { + deleted_vertices_.push_back(vertex->keys.Keys()); + } break; } - { - // We need to find the parent object in order to be able to use - // its lock. - auto parent = prev; - while (parent.type == PreviousPtr::Type::DELTA) { - parent = parent.delta->prev.Get(); - } - switch (parent.type) { - case PreviousPtr::Type::VERTEX: - case PreviousPtr::Type::EDGE: - break; - case PreviousPtr::Type::DELTA: - case PreviousPtr::Type::NULLPTR: - LOG_FATAL("Invalid database state!"); - } - } - Delta *prev_delta = prev.delta; - prev_delta->next.store(nullptr, std::memory_order_release); - break; - } - case PreviousPtr::Type::NULLPTR: { - LOG_FATAL("Invalid pointer!"); - } - } - break; - } - } + case PreviousPtr::Type::EDGE: { + Edge *edge = prev.edge; - unlinked_undo_buffers.emplace_back(0, std::move(transaction->deltas)); - committed_transactions_.pop_front(); + edge->delta = nullptr; + if (edge->deleted) { + deleted_edges_.push_back(edge->gid); + } + break; + } + case PreviousPtr::Type::DELTA: { + if (prev.delta->commit_info->start_or_commit_timestamp == commit_timestamp) { + // The delta that is newer than this one is also a delta from this + // transaction. We skip the current delta and will unlink it as a + // part of the suffix later. + break; + } + Delta *prev_delta = prev.delta; + prev_delta->next = nullptr; + break; + } + case PreviousPtr::Type::NULLPTR: { + LOG_FATAL("Invalid pointer!"); + } + } + break; + } + } + } else if (!transaction.is_aborted) { + Accessor(*this, transaction).Abort(); + } + it = start_logical_id_to_transaction_.erase(it); } // After unlinking deltas from vertices, we refresh the indices. That way // we're sure that none of the vertices from `current_deleted_vertices` // appears in an index, and we can safely remove the from the main storage // after the last currently active transaction is finished. - if (run_index_cleanup) { + if (cleaned_up_committed_transaction || has_any_transaction_aborted_since_last_gc) { // This operation is very expensive as it traverses through all of the items // in every index every time. - RemoveObsoleteEntries(&indices_, oldest_active_start_timestamp); - constraints_.unique_constraints.RemoveObsoleteEntries(oldest_active_start_timestamp); + RemoveObsoleteEntries(&indices_, clean_up_before_timestamp); } - { - uint64_t mark_timestamp = timestamp_; - for (auto &[timestamp, undo_buffer] : unlinked_undo_buffers) { - timestamp = mark_timestamp; - } - garbage_undo_buffers_.splice(garbage_undo_buffers_.end(), unlinked_undo_buffers); - - for (const auto &vertex : deleted_vertices_) { - garbage_vertices_.emplace_back(mark_timestamp, vertex); - } - } - - // if force is set to true we can simply delete all the leftover undos because - // no transaction is active - if constexpr (force) { - garbage_undo_buffers_.clear(); - } else { - while (!garbage_undo_buffers_.empty() && garbage_undo_buffers_.front().first <= oldest_active_start_timestamp) { - garbage_undo_buffers_.pop_front(); - } - } - - { - auto vertex_acc = vertices_.access(); - if constexpr (force) { - // if force is set to true, then we have unique_lock and no transactions are active - // so we can clean all of the deleted vertices - while (!garbage_vertices_.empty()) { - MG_ASSERT(vertex_acc.remove(garbage_vertices_.front().second), "Invalid database state!"); - garbage_vertices_.pop_front(); - } - } else { - while (!garbage_vertices_.empty() && garbage_vertices_.front().first < oldest_active_start_timestamp) { - MG_ASSERT(vertex_acc.remove(garbage_vertices_.front().second), "Invalid database state!"); - garbage_vertices_.pop_front(); - } - } + auto vertex_acc = vertices_.access(); + for (const auto &vertex : deleted_vertices_) { + MG_ASSERT(vertex_acc.remove(vertex), "Invalid database state!"); } + deleted_vertices_.clear(); { auto edge_acc = edges_.access(); for (auto edge : deleted_edges_) { MG_ASSERT(edge_acc.remove(edge), "Invalid database state!"); } } + deleted_edges_.clear(); } -// tell the linker he can find the CollectGarbage definitions here -template void Shard::CollectGarbage(); -template void Shard::CollectGarbage(); - -bool Shard::InitializeWalFile() { - if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL) - return false; - if (!wal_file_) { - wal_file_.emplace(wal_directory_, uuid_, epoch_id_, config_.items, &name_id_mapper_, wal_seq_num_++, - &file_retainer_); +Transaction &Shard::GetTransaction(const coordinator::Hlc start_timestamp, IsolationLevel isolation_level) { + // TODO(antaljanosbenjamin) + if (const auto it = start_logical_id_to_transaction_.find(start_timestamp.logical_id); + it != start_logical_id_to_transaction_.end()) { + MG_ASSERT(it->second->start_timestamp.coordinator_wall_clock == start_timestamp.coordinator_wall_clock, + "Logical id and wall clock don't match in already seen hybrid logical clock!"); + MG_ASSERT(it->second->isolation_level == isolation_level, + "Isolation level doesn't match in already existing transaction!"); + return *it->second; } - return true; -} - -void Shard::FinalizeWalFile() { - ++wal_unsynced_transactions_; - if (wal_unsynced_transactions_ >= config_.durability.wal_file_flush_every_n_tx) { - wal_file_->Sync(); - wal_unsynced_transactions_ = 0; - } - if (wal_file_->GetSize() / 1024 >= config_.durability.wal_file_size_kibibytes) { - wal_file_->FinalizeWal(); - wal_file_ = std::nullopt; - wal_unsynced_transactions_ = 0; - } else { - // Try writing the internal buffer if possible, if not - // the data should be written as soon as it's possible - // (triggered by the new transaction commit, or some - // reading thread EnabledFlushing) - wal_file_->TryFlushing(); - } -} - -void Shard::AppendToWal(const Transaction &transaction, uint64_t final_commit_timestamp) { - if (!InitializeWalFile()) return; - // Traverse deltas and append them to the WAL file. - // A single transaction will always be contained in a single WAL file. - auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire); - - if (replication_role_ == ReplicationRole::MAIN) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->StartTransactionReplication(wal_file_->SequenceNumber()); - } - }); - } - - // Helper lambda that traverses the delta chain on order to find the first - // delta that should be processed and then appends all discovered deltas. - auto find_and_apply_deltas = [&](const auto *delta, const auto &parent, auto filter) { - while (true) { - auto *older = delta->next.load(std::memory_order_acquire); - if (older == nullptr || older->timestamp->load(std::memory_order_acquire) != current_commit_timestamp) break; - delta = older; - } - while (true) { - if (filter(delta->action)) { - wal_file_->AppendDelta(*delta, parent, final_commit_timestamp); - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->IfStreamingTransaction( - [&](auto &stream) { stream.AppendDelta(*delta, parent, final_commit_timestamp); }); - } - }); - } - auto prev = delta->prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::DELTA) break; - delta = prev.delta; - } - }; - - // The deltas are ordered correctly in the `transaction.deltas` buffer, but we - // don't traverse them in that order. That is because for each delta we need - // information about the vertex or edge they belong to and that information - // isn't stored in the deltas themselves. In order to find out information - // about the corresponding vertex or edge it is necessary to traverse the - // delta chain for each delta until a vertex or edge is encountered. This - // operation is very expensive as the chain grows. - // Instead, we traverse the edges until we find a vertex or edge and traverse - // their delta chains. This approach has a drawback because we lose the - // correct order of the operations. Because of that, we need to traverse the - // deltas several times and we have to manually ensure that the stored deltas - // will be ordered correctly. - - // 1. Process all Vertex deltas and store all operations that create vertices - // and modify vertex data. - for (const auto &delta : transaction.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) continue; - find_and_apply_deltas(&delta, *prev.vertex, [](auto action) { - switch (action) { - case Delta::Action::DELETE_OBJECT: - case Delta::Action::SET_PROPERTY: - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - return true; - - case Delta::Action::RECREATE_OBJECT: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - return false; - } - }); - } - // 2. Process all Vertex deltas and store all operations that create edges. - for (const auto &delta : transaction.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) continue; - find_and_apply_deltas(&delta, *prev.vertex, [](auto action) { - switch (action) { - case Delta::Action::REMOVE_OUT_EDGE: - return true; - - case Delta::Action::DELETE_OBJECT: - case Delta::Action::RECREATE_OBJECT: - case Delta::Action::SET_PROPERTY: - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - return false; - } - }); - } - // 3. Process all Edge deltas and store all operations that modify edge data. - for (const auto &delta : transaction.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::EDGE) continue; - find_and_apply_deltas(&delta, *prev.edge, [](auto action) { - switch (action) { - case Delta::Action::SET_PROPERTY: - return true; - - case Delta::Action::DELETE_OBJECT: - case Delta::Action::RECREATE_OBJECT: - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - return false; - } - }); - } - // 4. Process all Vertex deltas and store all operations that delete edges. - for (const auto &delta : transaction.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) continue; - find_and_apply_deltas(&delta, *prev.vertex, [](auto action) { - switch (action) { - case Delta::Action::ADD_OUT_EDGE: - return true; - - case Delta::Action::DELETE_OBJECT: - case Delta::Action::RECREATE_OBJECT: - case Delta::Action::SET_PROPERTY: - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - return false; - } - }); - } - // 5. Process all Vertex deltas and store all operations that delete vertices. - for (const auto &delta : transaction.deltas) { - auto prev = delta.prev.Get(); - MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!"); - if (prev.type != PreviousPtr::Type::VERTEX) continue; - find_and_apply_deltas(&delta, *prev.vertex, [](auto action) { - switch (action) { - case Delta::Action::RECREATE_OBJECT: - return true; - - case Delta::Action::DELETE_OBJECT: - case Delta::Action::SET_PROPERTY: - case Delta::Action::ADD_LABEL: - case Delta::Action::REMOVE_LABEL: - case Delta::Action::ADD_IN_EDGE: - case Delta::Action::ADD_OUT_EDGE: - case Delta::Action::REMOVE_IN_EDGE: - case Delta::Action::REMOVE_OUT_EDGE: - return false; - } - }); - } - - // Add a delta that indicates that the transaction is fully written to the WAL - // file. - wal_file_->AppendTransactionEnd(final_commit_timestamp); - - FinalizeWalFile(); - - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(final_commit_timestamp); }); - client->FinalizeTransactionReplication(); - } - }); -} - -void Shard::AppendToWal(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t final_commit_timestamp) { - if (!InitializeWalFile()) return; - wal_file_->AppendOperation(operation, label, properties, final_commit_timestamp); - { - if (replication_role_ == ReplicationRole::MAIN) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->StartTransactionReplication(wal_file_->SequenceNumber()); - client->IfStreamingTransaction( - [&](auto &stream) { stream.AppendOperation(operation, label, properties, final_commit_timestamp); }); - client->FinalizeTransactionReplication(); - } - }); - } - } - FinalizeWalFile(); -} - -utils::BasicResult Shard::CreateSnapshot() { - if (replication_role_ != ReplicationRole::MAIN) { - return CreateSnapshotError::DisabledForReplica; - } - - // Create the transaction used to create the snapshot. - auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION); - - // Create snapshot. - // durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_, - // config_.durability.snapshot_retention_count, &vertices_, &edges_, - // &name_id_mapper_, &indices_, &constraints_, config_.items, schema_validator_, - // uuid_, epoch_id_, epoch_history_, &file_retainer_); - - // Finalize snapshot transaction. - commit_log_->MarkFinished(transaction.start_timestamp); - return {}; + const auto insert_result = start_logical_id_to_transaction_.emplace( + start_timestamp.logical_id, std::make_unique(start_timestamp, isolation_level)); + MG_ASSERT(insert_result.second, "Transaction creation failed!"); + return *insert_result.first->second; } void Shard::StoreMapping(std::unordered_map id_to_name) { name_id_mapper_.StoreMapping(std::move(id_to_name)); } -bool Shard::LockPath() { - auto locker_accessor = global_locker_.Access(); - return locker_accessor.AddPath(config_.durability.storage_directory); -} - -bool Shard::UnlockPath() { - { - auto locker_accessor = global_locker_.Access(); - if (!locker_accessor.RemovePath(config_.durability.storage_directory)) { - return false; - } - } - - // We use locker accessor in seperate scope so we don't produce deadlock - // after we call clean queue. - file_retainer_.CleanQueue(); - return true; -} - -void Shard::FreeMemory() { - CollectGarbage(); - - // SkipList is already threadsafe - vertices_.run_gc(); - edges_.run_gc(); - indices_.label_index.RunGC(); - indices_.label_property_index.RunGC(); -} - -uint64_t Shard::CommitTimestamp(const std::optional desired_commit_timestamp) { - if (!desired_commit_timestamp) { - return timestamp_++; - } - timestamp_ = std::max(timestamp_, *desired_commit_timestamp + 1); - return *desired_commit_timestamp; -} - bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const { return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ && (!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_); } -bool Shard::SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) { - // We don't want to restart the server if we're already a REPLICA - if (replication_role_ == ReplicationRole::REPLICA) { - return false; - } - - replication_server_ = std::make_unique(this, std::move(endpoint), config); - - replication_role_ = ReplicationRole::REPLICA; - return true; -} - -bool Shard::SetMainReplicationRole() { - // We don't want to generate new epoch_id and do the - // cleanup if we're already a MAIN - if (replication_role_ == ReplicationRole::MAIN) { - return false; - } - - // Main instance does not need replication server - // This should be always called first so we finalize everything - replication_server_.reset(nullptr); - - if (wal_file_) { - wal_file_->FinalizeWal(); - wal_file_.reset(); - } - - // Generate new epoch id and save the last one to the history. - if (epoch_history_.size() == kEpochHistoryRetention) { - epoch_history_.pop_front(); - } - epoch_history_.emplace_back(std::move(epoch_id_), last_commit_timestamp_); - epoch_id_ = utils::GenerateUUID(); - - replication_role_ = ReplicationRole::MAIN; - return true; -} - -utils::BasicResult Shard::RegisterReplica( - std::string name, io::network::Endpoint endpoint, const replication::ReplicationMode replication_mode, - const replication::ReplicationClientConfig &config) { - MG_ASSERT(replication_role_ == ReplicationRole::MAIN, "Only main instance can register a replica!"); - - const bool name_exists = replication_clients_.WithLock([&](auto &clients) { - return std::any_of(clients.begin(), clients.end(), [&name](const auto &client) { return client->Name() == name; }); - }); - - if (name_exists) { - return RegisterReplicaError::NAME_EXISTS; - } - - const auto end_point_exists = replication_clients_.WithLock([&endpoint](auto &clients) { - return std::any_of(clients.begin(), clients.end(), - [&endpoint](const auto &client) { return client->Endpoint() == endpoint; }); - }); - - if (end_point_exists) { - return RegisterReplicaError::END_POINT_EXISTS; - } - - MG_ASSERT(replication_mode == replication::ReplicationMode::SYNC || !config.timeout, - "Only SYNC mode can have a timeout set"); - - auto client = std::make_unique(std::move(name), this, endpoint, replication_mode, config); - if (client->State() == replication::ReplicaState::INVALID) { - return RegisterReplicaError::CONNECTION_FAILED; - } - - return replication_clients_.WithLock([&](auto &clients) -> utils::BasicResult { - // Another thread could have added a client with same name while - // we were connecting to this client. - if (std::any_of(clients.begin(), clients.end(), - [&](const auto &other_client) { return client->Name() == other_client->Name(); })) { - return RegisterReplicaError::NAME_EXISTS; - } - - if (std::any_of(clients.begin(), clients.end(), - [&client](const auto &other_client) { return client->Endpoint() == other_client->Endpoint(); })) { - return RegisterReplicaError::END_POINT_EXISTS; - } - - clients.push_back(std::move(client)); - return {}; - }); -} - -bool Shard::UnregisterReplica(const std::string_view name) { - MG_ASSERT(replication_role_ == ReplicationRole::MAIN, "Only main instance can unregister a replica!"); - return replication_clients_.WithLock([&](auto &clients) { - return std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); - }); -} - -std::optional Shard::GetReplicaState(const std::string_view name) { - return replication_clients_.WithLock([&](auto &clients) -> std::optional { - const auto client_it = - std::find_if(clients.cbegin(), clients.cend(), [name](auto &client) { return client->Name() == name; }); - if (client_it == clients.cend()) { - return std::nullopt; - } - return (*client_it)->State(); - }); -} - -ReplicationRole Shard::GetReplicationRole() const { return replication_role_; } - -std::vector Shard::ReplicasInfo() { - return replication_clients_.WithLock([](auto &clients) { - std::vector replica_info; - replica_info.reserve(clients.size()); - std::transform(clients.begin(), clients.end(), std::back_inserter(replica_info), - [](const auto &client) -> ReplicaInfo { - return {client->Name(), client->Mode(), client->Timeout(), client->Endpoint(), client->State()}; - }); - return replica_info; - }); -} - void Shard::SetIsolationLevel(IsolationLevel isolation_level) { isolation_level_ = isolation_level; } } // namespace memgraph::storage::v3 diff --git a/src/storage/v3/shard.hpp b/src/storage/v3/shard.hpp index 0a040cfff..8ebccf5c2 100644 --- a/src/storage/v3/shard.hpp +++ b/src/storage/v3/shard.hpp @@ -11,7 +11,6 @@ #pragma once -#include #include #include #include @@ -21,13 +20,11 @@ #include #include +#include "coordinator/hybrid_logical_clock.hpp" #include "io/network/endpoint.hpp" +#include "io/time.hpp" #include "kvstore/kvstore.hpp" -#include "storage/v3/commit_log.hpp" #include "storage/v3/config.hpp" -#include "storage/v3/constraints.hpp" -#include "storage/v3/durability/metadata.hpp" -#include "storage/v3/durability/wal.hpp" #include "storage/v3/edge.hpp" #include "storage/v3/edge_accessor.hpp" #include "storage/v3/id_types.hpp" @@ -46,6 +43,7 @@ #include "storage/v3/vertex_accessor.hpp" #include "storage/v3/vertex_id.hpp" #include "storage/v3/vertices_skip_list.hpp" +#include "storage/v3/view.hpp" #include "utils/exceptions.hpp" #include "utils/file_locker.hpp" #include "utils/on_scope_exit.hpp" @@ -55,13 +53,6 @@ #include "utils/synchronized.hpp" #include "utils/uuid.hpp" -/// REPLICATION /// -#include "rpc/server.hpp" -#include "storage/v3/replication/config.hpp" -#include "storage/v3/replication/enums.hpp" -#include "storage/v3/replication/rpc.hpp" -#include "storage/v3/replication/serialization.hpp" - namespace memgraph::storage::v3 { // The storage is based on this paper: @@ -78,7 +69,6 @@ class AllVerticesIterable final { Transaction *transaction_; View view_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; const Schemas *schemas_; @@ -102,13 +92,11 @@ class AllVerticesIterable final { }; AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view, - Indices *indices, Constraints *constraints, Config::Items config, - const VertexValidator &vertex_validator) + Indices *indices, Config::Items config, const VertexValidator &vertex_validator) : vertices_accessor_(std::move(vertices_accessor)), transaction_(transaction), view_(view), indices_(indices), - constraints_(constraints), config_(config), vertex_validator_{&vertex_validator} {} @@ -184,13 +172,6 @@ struct IndicesInfo { std::vector> label_property; }; -/// Structure used to return information about existing constraints in the -/// storage. -struct ConstraintsInfo { - std::vector> existence; - std::vector>> unique; -}; - /// Structure used to return information about existing schemas in the storage struct SchemasInfo { Schemas::SchemasList schemas; @@ -202,11 +183,8 @@ struct StorageInfo { uint64_t edge_count; double average_degree; uint64_t memory_usage; - uint64_t disk_usage; }; -enum class ReplicationRole : uint8_t { MAIN, REPLICA }; - class Shard final { public: /// @throw std::system_error @@ -224,19 +202,9 @@ class Shard final { private: friend class Shard; - explicit Accessor(Shard *shard, IsolationLevel isolation_level); + Accessor(Shard &shard, Transaction &transaction); public: - Accessor(const Accessor &) = delete; - Accessor &operator=(const Accessor &) = delete; - Accessor &operator=(Accessor &&other) = delete; - - // NOTE: After the accessor is moved, all objects derived from it (accessors - // and iterators) are *invalid*. You have to get all derived objects again. - Accessor(Accessor &&other) noexcept; - - ~Accessor(); - // TODO(gvolfing) this is just a workaround for stitching remove this later. LabelId GetPrimaryLabel() const noexcept { return shard_->primary_label_; } @@ -253,9 +221,8 @@ class Shard final { std::optional FindVertex(std::vector primary_key, View view); VerticesIterable Vertices(View view) { - return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), &transaction_, view, &shard_->indices_, - &shard_->constraints_, shard_->config_.items, - shard_->vertex_validator_)); + return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), transaction_, view, &shard_->indices_, + shard_->config_.items, shard_->vertex_validator_)); } VerticesIterable Vertices(LabelId label, View view); @@ -338,41 +305,28 @@ class Shard final { return {shard_->indices_.label_index.ListIndices(), shard_->indices_.label_property_index.ListIndices()}; } - ConstraintsInfo ListAllConstraints() const { - return {ListExistenceConstraints(shard_->constraints_), - shard_->constraints_.unique_constraints.ListConstraints()}; - } - const SchemaValidator &GetSchemaValidator() const; SchemasInfo ListAllSchemas() const { return {shard_->schemas_.ListSchemas()}; } void AdvanceCommand(); - /// Commit returns `ConstraintViolation` if the changes made by this - /// transaction violate an existence or unique constraint. In that case the - /// transaction is automatically aborted. Otherwise, void is returned. - /// @throw std::bad_alloc - utils::BasicResult Commit(std::optional desired_commit_timestamp = {}); + void Commit(coordinator::Hlc commit_timestamp); /// @throw std::bad_alloc void Abort(); - void FinalizeTransaction(); - private: /// @throw std::bad_alloc VertexAccessor CreateVertex(Gid gid, LabelId primary_label); Shard *shard_; - Transaction transaction_; - std::optional commit_timestamp_; - bool is_transaction_active_; + Transaction *transaction_; Config::Items config_; }; - Accessor Access(std::optional override_isolation_level = {}) { - return Accessor{this, override_isolation_level.value_or(isolation_level_)}; + Accessor Access(coordinator::Hlc start_timestamp, std::optional override_isolation_level = {}) { + return Accessor{*this, GetTransaction(start_timestamp, override_isolation_level.value_or(isolation_level_))}; } LabelId NameToLabel(std::string_view name) const; @@ -399,45 +353,6 @@ class Shard final { IndicesInfo ListAllIndices() const; - /// Creates an existence constraint. Returns true if the constraint was - /// successfully added, false if it already exists and a `ConstraintViolation` - /// if there is an existing vertex violating the constraint. - /// - /// @throw std::bad_alloc - /// @throw std::length_error - utils::BasicResult CreateExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp = {}); - - /// Removes an existence constraint. Returns true if the constraint was - /// removed, and false if it doesn't exist. - bool DropExistenceConstraint(LabelId label, PropertyId property, - std::optional desired_commit_timestamp = {}); - - /// Creates a unique constraint. In the case of two vertices violating the - /// constraint, it returns `ConstraintViolation`. Otherwise returns a - /// `UniqueConstraints::CreationStatus` enum with the following possibilities: - /// * `SUCCESS` if the constraint was successfully created, - /// * `ALREADY_EXISTS` if the constraint already existed, - /// * `EMPTY_PROPERTIES` if the property set is empty, or - // * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the - // limit of maximum number of properties. - /// - /// @throw std::bad_alloc - utils::BasicResult CreateUniqueConstraint( - LabelId label, const std::set &properties, std::optional desired_commit_timestamp = {}); - - /// Removes a unique constraint. Returns `UniqueConstraints::DeletionStatus` - /// enum with the following possibilities: - /// * `SUCCESS` if constraint was successfully removed, - /// * `NOT_FOUND` if the specified constraint was not found, - /// * `EMPTY_PROPERTIES` if the property set is empty, or - /// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the - // limit of maximum number of properties. - UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, const std::set &properties, - std::optional desired_commit_timestamp = {}); - - ConstraintsInfo ListAllConstraints() const; - SchemasInfo ListAllSchemas() const; const Schemas::Schema *GetSchema(LabelId primary_label) const; @@ -448,75 +363,15 @@ class Shard final { StorageInfo GetInfo() const; - bool LockPath(); - bool UnlockPath(); - - bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config = {}); - - bool SetMainReplicationRole(); - - enum class RegisterReplicaError : uint8_t { - NAME_EXISTS, - END_POINT_EXISTS, - CONNECTION_FAILED, - COULD_NOT_BE_PERSISTED - }; - - /// @pre The instance should have a MAIN role - /// @pre Timeout can only be set for SYNC replication - utils::BasicResult RegisterReplica( - std::string name, io::network::Endpoint endpoint, replication::ReplicationMode replication_mode, - const replication::ReplicationClientConfig &config = {}); - /// @pre The instance should have a MAIN role - bool UnregisterReplica(std::string_view name); - - std::optional GetReplicaState(std::string_view name); - - ReplicationRole GetReplicationRole() const; - - struct ReplicaInfo { - std::string name; - replication::ReplicationMode mode; - std::optional timeout; - io::network::Endpoint endpoint; - replication::ReplicaState state; - }; - - std::vector ReplicasInfo(); - - void FreeMemory(); - void SetIsolationLevel(IsolationLevel isolation_level); - enum class CreateSnapshotError : uint8_t { DisabledForReplica }; - - utils::BasicResult CreateSnapshot(); + // Might invalidate accessors + void CollectGarbage(io::Time current_time); void StoreMapping(std::unordered_map id_to_name); private: - Transaction CreateTransaction(IsolationLevel isolation_level); - - /// The force parameter determines the behaviour of the garbage collector. - /// If it's set to true, it will behave as a global operation, i.e. it can't - /// be part of a transaction, and no other transaction can be active at the same time. - /// This allows it to delete immediately vertices without worrying that some other - /// transaction is possibly using it. If there are active transactions when this method - /// is called with force set to true, it will fallback to the same method with the force - /// set to false. - /// If it's set to false, it will execute in parallel with other transactions, ensuring - /// that no object in use can be deleted. - /// @throw std::system_error - /// @throw std::bad_alloc - template - void CollectGarbage(); - - bool InitializeWalFile(); - void FinalizeWalFile(); - - void AppendToWal(const Transaction &transaction, uint64_t final_commit_timestamp); - void AppendToWal(durability::StorageGlobalOperation operation, LabelId label, const std::set &properties, - uint64_t final_commit_timestamp); + Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level); uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); @@ -537,45 +392,22 @@ class Shard final { SchemaValidator schema_validator_; VertexValidator vertex_validator_; - Constraints constraints_; Indices indices_; Schemas schemas_; - // Transaction engine - uint64_t timestamp_{kTimestampInitialId}; - uint64_t transaction_id_{kTransactionInitialId}; - // TODO: This isn't really a commit log, it doesn't even care if a - // transaction commited or aborted. We could probably combine this with - // `timestamp_` in a sensible unit, something like TransactionClock or - // whatever. - std::optional commit_log_; - - std::list committed_transactions_; + std::list committed_transactions_; IsolationLevel isolation_level_; Config config_; - // Undo buffers that were unlinked and now are waiting to be freed. - std::list>> garbage_undo_buffers_; - // Vertices that are logically deleted but still have to be removed from // indices before removing them from the main storage. std::list deleted_vertices_; - // Vertices that are logically deleted and removed from indices and now wait - // to be removed from the main storage. - std::list> garbage_vertices_; - // Edges that are logically deleted and wait to be removed from the main // storage. std::list deleted_edges_; - // Durability - std::filesystem::path snapshot_directory_; - std::filesystem::path wal_directory_; - std::filesystem::path lock_file_path_; - utils::OutputFile lock_file_handle_; - // UUID used to distinguish snapshots and to link snapshots to WALs std::string uuid_; // Sequence number used to keep track of the chain of WALs. @@ -601,7 +433,6 @@ class Shard final { // epoch. std::deque> epoch_history_; - std::optional wal_file_; uint64_t wal_unsynced_transactions_{0}; utils::FileRetainer file_retainer_; @@ -609,27 +440,10 @@ class Shard final { // Global locker that is used for clients file locking utils::FileRetainer::FileLocker global_locker_; - // Last commited timestamp - uint64_t last_commit_timestamp_{kTimestampInitialId}; - - class ReplicationServer; - std::unique_ptr replication_server_{nullptr}; - - class ReplicationClient; - // We create ReplicationClient using unique_ptr so we can move - // newly created client into the vector. - // We cannot move the client directly because it contains ThreadPool - // which cannot be moved. Also, the move is necessary because - // we don't want to create the client directly inside the vector - // because that would require the lock on the list putting all - // commits (they iterate list of clients) to halt. - // This way we can initialize client in main thread which means - // that we can immediately notify the user if the initialization - // failed. - using ReplicationClientList = utils::Synchronized>, utils::SpinLock>; - ReplicationClientList replication_clients_; - - ReplicationRole replication_role_{ReplicationRole::MAIN}; + // Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but + // haven't been cleaned up yet + std::map> start_logical_id_to_transaction_{}; + bool has_any_transaction_aborted_since_last_gc{false}; }; } // namespace memgraph::storage::v3 diff --git a/src/storage/v3/shard_rsm.cpp b/src/storage/v3/shard_rsm.cpp index 72b6f62df..d2d1423dd 100644 --- a/src/storage/v3/shard_rsm.cpp +++ b/src/storage/v3/shard_rsm.cpp @@ -12,6 +12,7 @@ #include #include +#include "query/v2/requests.hpp" #include "storage/v3/shard_rsm.hpp" #include "storage/v3/vertex_accessor.hpp" @@ -193,7 +194,7 @@ Value ConstructValueVertex(const memgraph::storage::v3::VertexAccessor &acc, mem namespace memgraph::storage::v3 { msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) { - auto acc = shard_->Access(); + auto acc = shard_->Access(req.transaction_id); // Workaround untill we have access to CreateVertexAndValidate() // with the new signature that does not require the primary label. @@ -243,23 +244,12 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) { } } - msgs::CreateVerticesResponse resp{}; - resp.success = action_successful; - - if (action_successful) { - auto result = acc.Commit(req.transaction_id.logical_id); - if (result.HasError()) { - resp.success = false; - spdlog::debug(&"ConstraintViolation, commiting vertices was unsuccesfull with transaction id: "[req.transaction_id - .logical_id]); - } - } - return resp; + return msgs::CreateVerticesResponse{action_successful}; } msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) { bool action_successful = true; - auto acc = shard_->Access(); + auto acc = shard_->Access(req.transaction_id); for (auto &propval : req.primary_keys) { if (!action_successful) { @@ -269,9 +259,8 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) { auto vertex_acc = acc.FindVertex(ConvertPropertyVector(std::move(propval)), View::OLD); if (!vertex_acc) { - spdlog::debug( - &"Error while trying to delete vertex. Vertex to delete does not exist. Transaction id: "[req.transaction_id - .logical_id]); + spdlog::debug("Error while trying to delete vertex. Vertex to delete does not exist. Transaction id: {}", + req.transaction_id.logical_id); action_successful = false; } else { // TODO(gvolfing) @@ -282,7 +271,7 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) { auto result = acc.DeleteVertex(&vertex_acc.value()); if (result.HasError() || !(result.GetValue().has_value())) { action_successful = false; - spdlog::debug(&"Error while trying to delete vertex. Transaction id: "[req.transaction_id.logical_id]); + spdlog::debug("Error while trying to delete vertex. Transaction id: {}", req.transaction_id.logical_id); } break; @@ -291,8 +280,8 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) { auto result = acc.DetachDeleteVertex(&vertex_acc.value()); if (result.HasError() || !(result.GetValue().has_value())) { action_successful = false; - spdlog::debug( - &"Error while trying to detach and delete vertex. Transaction id: "[req.transaction_id.logical_id]); + spdlog::debug("Error while trying to detach and delete vertex. Transaction id: {}", + req.transaction_id.logical_id); } break; @@ -301,23 +290,11 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) { } } - msgs::DeleteVerticesResponse resp{}; - resp.success = action_successful; - - if (action_successful) { - auto result = acc.Commit(req.transaction_id.logical_id); - if (result.HasError()) { - resp.success = false; - spdlog::debug(&"ConstraintViolation, commiting vertices was unsuccesfull with transaction id: "[req.transaction_id - .logical_id]); - } - } - - return resp; + return msgs::DeleteVerticesResponse{action_successful}; } msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateEdgesRequest &&req) { - auto acc = shard_->Access(); + auto acc = shard_->Access(req.transaction_id); bool action_successful = true; for (auto &edge : req.edges) { @@ -329,8 +306,8 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateEdgesRequest &&req) { if (!vertex_from_acc || !vertex_to_acc) { action_successful = false; - spdlog::debug( - &"Error while trying to insert edge, vertex does not exist. Transaction id: "[req.transaction_id.logical_id]); + spdlog::debug("Error while trying to insert edge, vertex does not exist. Transaction id: {}", + req.transaction_id.logical_id); break; } @@ -341,30 +318,16 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateEdgesRequest &&req) { if (edge_acc.HasError()) { action_successful = false; - spdlog::debug(&"Creating edge was not successful. Transaction id: "[req.transaction_id.logical_id]); + spdlog::debug("Creating edge was not successful. Transaction id: {}", req.transaction_id.logical_id); break; } } - msgs::CreateEdgesResponse resp{}; - - resp.success = action_successful; - - if (action_successful) { - auto result = acc.Commit(req.transaction_id.logical_id); - if (result.HasError()) { - resp.success = false; - spdlog::debug( - &"ConstraintViolation, commiting edge creation was unsuccesfull with transaction id: "[req.transaction_id - .logical_id]); - } - } - - return resp; + return msgs::CreateEdgesResponse{action_successful}; } msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) { - auto acc = shard_->Access(); + auto acc = shard_->Access(req.transaction_id); bool action_successful = true; std::vector results; @@ -421,6 +384,11 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) { return resp; } +msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CommitRequest &&req) { + shard_->Access(req.transaction_id).Commit(req.commit_timestamp); + return msgs::CommitResponse{true}; +}; + // NOLINTNEXTLINE(readability-convert-member-functions-to-static) msgs::WriteResponses ShardRsm::ApplyWrite(msgs::UpdateVerticesRequest && /*req*/) { return msgs::UpdateVerticesResponse{}; diff --git a/src/storage/v3/shard_rsm.hpp b/src/storage/v3/shard_rsm.hpp index 16a8a3b7d..ee1f11128 100644 --- a/src/storage/v3/shard_rsm.hpp +++ b/src/storage/v3/shard_rsm.hpp @@ -14,7 +14,6 @@ #include #include -#include #include "query/v2/requests.hpp" #include "storage/v3/shard.hpp" #include "storage/v3/vertex_accessor.hpp" @@ -39,6 +38,8 @@ class ShardRsm { msgs::WriteResponses ApplyWrite(msgs::DeleteEdgesRequest &&req); msgs::WriteResponses ApplyWrite(msgs::UpdateEdgesRequest &&req); + msgs::WriteResponses ApplyWrite(msgs::CommitRequest &&req); + public: explicit ShardRsm(std::unique_ptr &&shard) : shard_(std::move(shard)){}; diff --git a/src/storage/v3/transaction.hpp b/src/storage/v3/transaction.hpp index be92d1ef9..229e071b7 100644 --- a/src/storage/v3/transaction.hpp +++ b/src/storage/v3/transaction.hpp @@ -11,13 +11,11 @@ #pragma once -#include #include #include #include -#include "utils/skip_list.hpp" - +#include "coordinator/hybrid_logical_clock.hpp" #include "storage/v3/delta.hpp" #include "storage/v3/edge.hpp" #include "storage/v3/isolation_level.hpp" @@ -27,24 +25,27 @@ namespace memgraph::storage::v3 { -const uint64_t kTimestampInitialId = 0; -const uint64_t kTransactionInitialId = 1ULL << 63U; +struct CommitInfo { + bool is_locally_committed{false}; + coordinator::Hlc start_or_commit_timestamp; +}; struct Transaction { - Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level) - : transaction_id(transaction_id), - start_timestamp(start_timestamp), + Transaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level) + : start_timestamp(start_timestamp), + commit_info(std::make_unique(CommitInfo{false, {start_timestamp}})), command_id(0), must_abort(false), + is_aborted(false), isolation_level(isolation_level) {} Transaction(Transaction &&other) noexcept - : transaction_id(other.transaction_id), - start_timestamp(other.start_timestamp), - commit_timestamp(std::move(other.commit_timestamp)), + : start_timestamp(other.start_timestamp), + commit_info(std::move(other.commit_info)), command_id(other.command_id), deltas(std::move(other.deltas)), must_abort(other.must_abort), + is_aborted(other.is_aborted), isolation_level(other.isolation_level) {} Transaction(const Transaction &) = delete; @@ -53,32 +54,23 @@ struct Transaction { ~Transaction() {} - /// @throw std::bad_alloc if failed to create the `commit_timestamp` - void EnsureCommitTimestampExists() { - if (commit_timestamp != nullptr) return; - commit_timestamp = std::make_unique>(transaction_id); - } - - uint64_t transaction_id; - uint64_t start_timestamp; - // The `Transaction` object is stack allocated, but the `commit_timestamp` - // must be heap allocated because `Delta`s have a pointer to it, and that - // pointer must stay valid after the `Transaction` is moved into - // `commited_transactions_` list for GC. - std::unique_ptr> commit_timestamp; + coordinator::Hlc start_timestamp; + std::unique_ptr commit_info; uint64_t command_id; std::list deltas; bool must_abort; + bool is_aborted; IsolationLevel isolation_level; }; +// Relies on start timestamps are unique inline bool operator==(const Transaction &first, const Transaction &second) { - return first.transaction_id == second.transaction_id; + return first.start_timestamp == second.start_timestamp; } inline bool operator<(const Transaction &first, const Transaction &second) { - return first.transaction_id < second.transaction_id; + return first.start_timestamp < second.start_timestamp; } -inline bool operator==(const Transaction &first, const uint64_t &second) { return first.transaction_id == second; } -inline bool operator<(const Transaction &first, const uint64_t &second) { return first.transaction_id < second; } +inline bool operator==(const Transaction &first, const uint64_t second) { return first.start_timestamp == second; } +inline bool operator<(const Transaction &first, const uint64_t second) { return first.start_timestamp < second; } } // namespace memgraph::storage::v3 diff --git a/src/storage/v3/vertex_accessor.cpp b/src/storage/v3/vertex_accessor.cpp index e0a3b3279..68500c332 100644 --- a/src/storage/v3/vertex_accessor.cpp +++ b/src/storage/v3/vertex_accessor.cpp @@ -64,13 +64,13 @@ std::pair IsVisible(Vertex *vertex, Transaction *transaction, View v } // namespace detail std::optional VertexAccessor::Create(Vertex *vertex, Transaction *transaction, Indices *indices, - Constraints *constraints, Config::Items config, - const VertexValidator &vertex_validator, View view) { + Config::Items config, const VertexValidator &vertex_validator, + View view) { if (const auto [exists, deleted] = detail::IsVisible(vertex, transaction, view); !exists || deleted) { return std::nullopt; } - return VertexAccessor{vertex, transaction, indices, constraints, config, vertex_validator}; + return VertexAccessor{vertex, transaction, indices, config, vertex_validator}; } bool VertexAccessor::IsVisible(View view) const { @@ -544,7 +544,7 @@ Result> VertexAccessor::InEdges(View view, const std:: const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()}; for (const auto &item : in_edges) { const auto &[edge_type, from_vertex, edge] = item; - ret.emplace_back(edge, edge_type, from_vertex, id, transaction_, indices_, constraints_, config_); + ret.emplace_back(edge, edge_type, from_vertex, id, transaction_, indices_, config_); } return ret; } @@ -624,7 +624,7 @@ Result> VertexAccessor::OutEdges(View view, const std: const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()}; for (const auto &item : out_edges) { const auto &[edge_type, to_vertex, edge] = item; - ret.emplace_back(edge, edge_type, id, to_vertex, transaction_, indices_, constraints_, config_); + ret.emplace_back(edge, edge_type, id, to_vertex, transaction_, indices_, config_); } return ret; } diff --git a/src/storage/v3/vertex_accessor.hpp b/src/storage/v3/vertex_accessor.hpp index 2ba00a87b..da576845c 100644 --- a/src/storage/v3/vertex_accessor.hpp +++ b/src/storage/v3/vertex_accessor.hpp @@ -28,7 +28,6 @@ namespace memgraph::storage::v3 { class EdgeAccessor; class Shard; struct Indices; -struct Constraints; class VertexAccessor final { private: @@ -37,19 +36,17 @@ class VertexAccessor final { public: // Be careful when using VertexAccessor since it can be instantiated with // nullptr values - VertexAccessor(Vertex *vertex, Transaction *transaction, Indices *indices, Constraints *constraints, - Config::Items config, const VertexValidator &vertex_validator, bool for_deleted = false) + VertexAccessor(Vertex *vertex, Transaction *transaction, Indices *indices, Config::Items config, + const VertexValidator &vertex_validator, bool for_deleted = false) : vertex_(vertex), transaction_(transaction), indices_(indices), - constraints_(constraints), config_(config), vertex_validator_{&vertex_validator}, for_deleted_(for_deleted) {} static std::optional Create(Vertex *vertex, Transaction *transaction, Indices *indices, - Constraints *constraints, Config::Items config, - const VertexValidator &vertex_validator, View view); + Config::Items config, const VertexValidator &vertex_validator, View view); /// @return true if the object is visible from the current transaction bool IsVisible(View view) const; @@ -141,7 +138,6 @@ class VertexAccessor final { Vertex *vertex_; Transaction *transaction_; Indices *indices_; - Constraints *constraints_; Config::Items config_; const VertexValidator *vertex_validator_; diff --git a/src/utils/logging.hpp b/src/utils/logging.hpp index 87517b05a..de28b6ce4 100644 --- a/src/utils/logging.hpp +++ b/src/utils/logging.hpp @@ -37,7 +37,7 @@ namespace memgraph::logging { // compilers inline void AssertFailed(const char *file_name, int line_num, const char *expr, const std::string &message) { spdlog::critical( - "\nAssertion failed in file {} at line {}." + "\nAssertion failed at {}:{}." "\n\tExpression: '{}'" "{}", file_name, line_num, expr, !message.empty() ? fmt::format("\n\tMessage: '{}'", message) : ""); diff --git a/tests/simulation/shard_rsm.cpp b/tests/simulation/shard_rsm.cpp index f7a0aa4a8..d40346505 100644 --- a/tests/simulation/shard_rsm.cpp +++ b/tests/simulation/shard_rsm.cpp @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include #include #include #include @@ -126,17 +127,15 @@ bool AttemtpToCreateVertex(ShardClient &client, int64_t value) { create_req.new_vertices = {vertex}; create_req.transaction_id.logical_id = GetTransactionId(); - while (true) { - auto write_res = client.SendWriteRequest(create_req); - if (write_res.HasError()) { - continue; - } + auto write_res = client.SendWriteRequest(create_req); + MG_ASSERT(write_res.HasValue() && std::get(write_res.GetValue()).success, + "Unexpected failure"); - auto write_response_result = write_res.GetValue(); - auto write_response = std::get(write_response_result); - - return write_response.success; - } + auto commit_req = msgs::CommitRequest{create_req.transaction_id, msgs::Hlc{.logical_id = GetTransactionId()}}; + auto commit_res = client.SendWriteRequest(commit_req); + MG_ASSERT(commit_res.HasValue() && std::get(commit_res.GetValue()).success, + "Unexpected failure"); + return true; } bool AttemptToAddEdge(ShardClient &client, int64_t value_of_vertex_1, int64_t value_of_vertex_2, int64_t edge_gid, @@ -161,17 +160,15 @@ bool AttemptToAddEdge(ShardClient &client, int64_t value_of_vertex_1, int64_t va create_req.edges = {edge}; create_req.transaction_id.logical_id = GetTransactionId(); - while (true) { - auto write_res = client.SendWriteRequest(create_req); - if (write_res.HasError()) { - continue; - } + auto write_res = client.SendWriteRequest(create_req); + MG_ASSERT(write_res.HasValue() && std::get(write_res.GetValue()).success, + "Unexpected failure"); - auto write_response_result = write_res.GetValue(); - auto write_response = std::get(write_response_result); - - return write_response.success; - } + auto commit_req = msgs::CommitRequest{create_req.transaction_id, msgs::Hlc{.logical_id = GetTransactionId()}}; + auto commit_res = client.SendWriteRequest(commit_req); + MG_ASSERT(commit_res.HasValue() && std::get(commit_res.GetValue()).success, + "Unexpected failure"); + return true; } std::tuple> AttemptToScanAllWithBatchLimit(ShardClient &client, @@ -298,12 +295,16 @@ int TestMessages() { auto simulator = Simulator(config); Io shard_server_io_1 = simulator.RegisterNew(); + shard_server_io_1.SetDefaultTimeout(std::chrono::seconds(1)); const auto shard_server_1_address = shard_server_io_1.GetAddress(); Io shard_server_io_2 = simulator.RegisterNew(); + shard_server_io_2.SetDefaultTimeout(std::chrono::seconds(1)); const auto shard_server_2_address = shard_server_io_2.GetAddress(); Io shard_server_io_3 = simulator.RegisterNew(); + shard_server_io_3.SetDefaultTimeout(std::chrono::seconds(1)); const auto shard_server_3_address = shard_server_io_3.GetAddress(); Io shard_client_io = simulator.RegisterNew(); + shard_client_io.SetDefaultTimeout(std::chrono::seconds(1)); PropertyValue min_pk(static_cast(0)); std::vector min_prim_key = {min_pk}; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 5b6a2fad7..0540aacef 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -346,6 +346,9 @@ target_link_libraries(${test_prefix}storage_v3_vertex_accessors mg-storage-v3) add_unit_test(storage_v3_edge.cpp) target_link_libraries(${test_prefix}storage_v3_edge mg-storage-v3) +add_unit_test(storage_v3_isolation_level.cpp) +target_link_libraries(${test_prefix}storage_v3_isolation_level mg-storage-v3) + # Test mg-query-v2 add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp) target_link_libraries(${test_prefix}query_v2_interpreter mg-storage-v3 mg-query-v2 mg-communication) diff --git a/tests/unit/storage_v3.cpp b/tests/unit/storage_v3.cpp index 5efe2005e..944ef58bc 100644 --- a/tests/unit/storage_v3.cpp +++ b/tests/unit/storage_v3.cpp @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include #include #include @@ -16,6 +17,8 @@ #include #include +#include "coordinator/hybrid_logical_clock.hpp" +#include "io/time.hpp" #include "storage/v3/delta.hpp" #include "storage/v3/id_types.hpp" #include "storage/v3/key_store.hpp" @@ -32,7 +35,7 @@ using testing::UnorderedElementsAre; namespace memgraph::storage::v3::tests { -class StorageV3 : public ::testing::Test { +class StorageV3 : public ::testing::TestWithParam { protected: void SetUp() override { store.StoreMapping({{1, "label"}, {2, "property"}}); @@ -40,6 +43,8 @@ class StorageV3 : public ::testing::Test { store.CreateSchema(primary_label, {storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}})); } + void TearDown() override { CleanupHlc(last_hlc); } + VertexAccessor CreateVertexAndValidate(Shard::Accessor &acc, LabelId primary_label, const std::vector &labels, const std::vector> &properties) { @@ -54,33 +59,67 @@ class StorageV3 : public ::testing::Test { EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) { return store.NameToEdgeType(edge_type_name); } + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += wall_clock_increment; + return last_hlc; + } + + void CleanupHlc(const coordinator::Hlc hlc) { + if (with_gc) { + store.CollectGarbage(hlc.coordinator_wall_clock + reclamation_interval + one_time_unit); + } + } + + const bool with_gc = GetParam(); + static constexpr std::chrono::seconds wall_clock_increment{10}; + static constexpr std::chrono::seconds reclamation_interval{wall_clock_increment / 2}; + static constexpr io::Duration one_time_unit{1}; const std::vector pk{PropertyValue{0}}; const LabelId primary_label{LabelId::FromUint(1)}; - Shard store{primary_label, pk, std::nullopt}; const PropertyId primary_property{PropertyId::FromUint(2)}; + Shard store{primary_label, pk, std::nullopt, Config{.gc = {.reclamation_interval = reclamation_interval}}}; + coordinator::Hlc last_hlc{0, io::Time{}}; }; +INSTANTIATE_TEST_CASE_P(WithGc, StorageV3, ::testing::Values(true)); +INSTANTIATE_TEST_CASE_P(WithoutGc, StorageV3, ::testing::Values(false)); // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, Commit) { +TEST_P(StorageV3, Commit) { + const auto test_vertex_exists = [this](const coordinator::Hlc hlc) { + auto acc = store.Access(hlc); + ASSERT_TRUE(acc.FindVertex(pk, View::OLD).has_value()); + EXPECT_EQ(CountVertices(acc, View::OLD), 1U); + ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); + EXPECT_EQ(CountVertices(acc, View::NEW), 1U); + }; + const auto test_vertex_not_exists = [this](const coordinator::Hlc hlc) { + auto acc = store.Access(GetNextHlc()); + ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); + EXPECT_EQ(CountVertices(acc, View::OLD), 0U); + ASSERT_FALSE(acc.FindVertex(pk, View::NEW).has_value()); + EXPECT_EQ(CountVertices(acc, View::NEW), 0U); + }; + + const auto create_start_hlc = GetNextHlc(); { - auto acc = store.Access(); + auto acc = store.Access(create_start_hlc); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); EXPECT_EQ(CountVertices(acc, View::NEW), 1U); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } + const auto after_create_hlc = GetNextHlc(); + + ASSERT_NO_FATAL_FAILURE(test_vertex_exists(GetNextHlc())); + CleanupHlc(create_start_hlc); + ASSERT_NO_FATAL_FAILURE(test_vertex_exists(GetNextHlc())); + + const auto delete_start_hlc = GetNextHlc(); { - auto acc = store.Access(); - ASSERT_TRUE(acc.FindVertex(pk, View::OLD).has_value()); - EXPECT_EQ(CountVertices(acc, View::OLD), 1U); - ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); - EXPECT_EQ(CountVertices(acc, View::NEW), 1U); - acc.Abort(); - } - { - auto acc = store.Access(); + auto acc = store.Access(delete_start_hlc); auto vertex = acc.FindVertex(pk, View::NEW); ASSERT_TRUE(vertex); @@ -93,22 +132,27 @@ TEST_F(StorageV3, Commit) { EXPECT_EQ(CountVertices(acc, View::OLD), 0U); EXPECT_EQ(CountVertices(acc, View::NEW), 0U); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } - { - auto acc = store.Access(); - ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); - EXPECT_EQ(CountVertices(acc, View::OLD), 0U); - ASSERT_FALSE(acc.FindVertex(pk, View::NEW).has_value()); - EXPECT_EQ(CountVertices(acc, View::NEW), 0U); - acc.Abort(); + + ASSERT_NO_FATAL_FAILURE(test_vertex_not_exists(GetNextHlc())); + // The delta about deleting the vertex is still there, it + ASSERT_NO_FATAL_FAILURE(test_vertex_exists(after_create_hlc)); + CleanupHlc(delete_start_hlc); + + ASSERT_NO_FATAL_FAILURE(test_vertex_not_exists(GetNextHlc())); + + if (with_gc) { + ASSERT_NO_FATAL_FAILURE(test_vertex_not_exists(create_start_hlc)); + } else { + ASSERT_NO_FATAL_FAILURE(test_vertex_exists(after_create_hlc)); } } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, Abort) { +TEST_P(StorageV3, Abort) { { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); @@ -117,7 +161,7 @@ TEST_F(StorageV3, Abort) { acc.Abort(); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); ASSERT_FALSE(acc.FindVertex(pk, View::NEW).has_value()); @@ -127,12 +171,37 @@ TEST_F(StorageV3, Abort) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, AdvanceCommandCommit) { +TEST_P(StorageV3, AbortByGc) { + if (!with_gc) { + return; + } + { + const auto hlc = GetNextHlc(); + auto acc = store.Access(hlc); + CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); + ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); + EXPECT_EQ(CountVertices(acc, View::OLD), 0U); + ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); + EXPECT_EQ(CountVertices(acc, View::NEW), 1U); + CleanupHlc(hlc); + } + { + auto acc = store.Access(GetNextHlc()); + ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); + EXPECT_EQ(CountVertices(acc, View::OLD), 0U); + ASSERT_FALSE(acc.FindVertex(pk, View::NEW).has_value()); + EXPECT_EQ(CountVertices(acc, View::NEW), 0U); + acc.Abort(); + } +} + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(StorageV3, AdvanceCommandCommit) { std::vector pk1{PropertyValue{0}}; std::vector pk2{PropertyValue(2)}; { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk1, View::OLD).has_value()); @@ -151,10 +220,11 @@ TEST_F(StorageV3, AdvanceCommandCommit) { ASSERT_TRUE(acc.FindVertex(pk1, View::OLD).has_value()); ASSERT_TRUE(acc.FindVertex(pk1, View::NEW).has_value()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } + CleanupHlc(last_hlc); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); ASSERT_TRUE(acc.FindVertex(pk1, View::OLD).has_value()); ASSERT_TRUE(acc.FindVertex(pk1, View::NEW).has_value()); ASSERT_TRUE(acc.FindVertex(pk2, View::OLD).has_value()); @@ -166,11 +236,11 @@ TEST_F(StorageV3, AdvanceCommandCommit) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, AdvanceCommandAbort) { +TEST_P(StorageV3, AdvanceCommandAbort) { std::vector pk1{PropertyValue{0}}; std::vector pk2{PropertyValue(2)}; { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk1, View::OLD).has_value()); @@ -191,8 +261,8 @@ TEST_F(StorageV3, AdvanceCommandAbort) { acc.Abort(); } - { - auto acc = store.Access(); + const auto check_vertex_not_exists = [this, &pk1, &pk2]() { + auto acc = store.Access(GetNextHlc()); ASSERT_FALSE(acc.FindVertex(pk1, View::OLD).has_value()); ASSERT_FALSE(acc.FindVertex(pk1, View::NEW).has_value()); ASSERT_FALSE(acc.FindVertex(pk2, View::OLD).has_value()); @@ -200,13 +270,17 @@ TEST_F(StorageV3, AdvanceCommandAbort) { EXPECT_EQ(CountVertices(acc, View::OLD), 0U); EXPECT_EQ(CountVertices(acc, View::NEW), 0U); acc.Abort(); - } + }; + ASSERT_NO_FATAL_FAILURE(check_vertex_not_exists()); + CleanupHlc(last_hlc); + ASSERT_NO_FATAL_FAILURE(check_vertex_not_exists()); } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, SnapshotIsolation) { - auto acc1 = store.Access(); - auto acc2 = store.Access(); +TEST_P(StorageV3, SnapshotIsolation) { + const auto start_hlc1 = GetNextHlc(); + auto acc1 = store.Access(start_hlc1); + auto acc2 = store.Access(GetNextHlc()); CreateVertexAndValidate(acc1, primary_label, {}, {{primary_property, PropertyValue{0}}}); @@ -217,7 +291,7 @@ TEST_F(StorageV3, SnapshotIsolation) { EXPECT_EQ(CountVertices(acc1, View::NEW), 1U); EXPECT_EQ(CountVertices(acc2, View::NEW), 0U); - ASSERT_FALSE(acc1.Commit().HasError()); + acc1.Commit(GetNextHlc()); ASSERT_FALSE(acc2.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc2, View::OLD), 0U); @@ -226,18 +300,19 @@ TEST_F(StorageV3, SnapshotIsolation) { acc2.Abort(); - auto acc3 = store.Access(); + auto acc3 = store.Access(GetNextHlc()); ASSERT_TRUE(acc3.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc3, View::OLD), 1U); + CleanupHlc(start_hlc1); ASSERT_TRUE(acc3.FindVertex(pk, View::NEW).has_value()); EXPECT_EQ(CountVertices(acc3, View::NEW), 1U); acc3.Abort(); } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, AccessorMove) { +TEST_P(StorageV3, AccessorMove) { { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); @@ -252,10 +327,10 @@ TEST_F(StorageV3, AccessorMove) { ASSERT_TRUE(moved.FindVertex(pk, View::NEW).has_value()); EXPECT_EQ(CountVertices(moved, View::NEW), 1U); - ASSERT_FALSE(moved.Commit().HasError()); + moved.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); ASSERT_TRUE(acc.FindVertex(pk, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 1U); ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); @@ -265,9 +340,9 @@ TEST_F(StorageV3, AccessorMove) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexDeleteCommit) { - auto acc1 = store.Access(); // read transaction - auto acc2 = store.Access(); // write transaction +TEST_P(StorageV3, VertexDeleteCommit) { + auto acc1 = store.Access(GetNextHlc()); // read transaction + auto acc2 = store.Access(GetNextHlc()); // write transaction // Create the vertex in transaction 2 { @@ -276,11 +351,11 @@ TEST_F(StorageV3, VertexDeleteCommit) { EXPECT_EQ(CountVertices(acc2, View::OLD), 0U); ASSERT_TRUE(acc2.FindVertex(pk, View::NEW).has_value()); EXPECT_EQ(CountVertices(acc2, View::NEW), 1U); - ASSERT_FALSE(acc2.Commit().HasError()); + acc2.Commit(GetNextHlc()); } - auto acc3 = store.Access(); // read transaction - auto acc4 = store.Access(); // write transaction + auto acc3 = store.Access(GetNextHlc()); // read transaction + auto acc4 = store.Access(GetNextHlc()); // write transaction // Check whether the vertex exists in transaction 1 ASSERT_FALSE(acc1.FindVertex(pk, View::OLD).has_value()); @@ -310,10 +385,10 @@ TEST_F(StorageV3, VertexDeleteCommit) { EXPECT_EQ(CountVertices(acc4, View::OLD), 0U); EXPECT_EQ(CountVertices(acc4, View::NEW), 0U); - ASSERT_FALSE(acc4.Commit().HasError()); + acc4.Commit(GetNextHlc()); } - auto acc5 = store.Access(); // read transaction + auto acc5 = store.Access(GetNextHlc()); // read transaction // Check whether the vertex exists in transaction 1 ASSERT_FALSE(acc1.FindVertex(pk, View::OLD).has_value()); @@ -335,9 +410,9 @@ TEST_F(StorageV3, VertexDeleteCommit) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexDeleteAbort) { - auto acc1 = store.Access(); // read transaction - auto acc2 = store.Access(); // write transaction +TEST_P(StorageV3, VertexDeleteAbort) { + auto acc1 = store.Access(GetNextHlc()); // read transaction + auto acc2 = store.Access(GetNextHlc()); // write transaction // Create the vertex in transaction 2 { @@ -346,11 +421,11 @@ TEST_F(StorageV3, VertexDeleteAbort) { EXPECT_EQ(CountVertices(acc2, View::OLD), 0U); ASSERT_TRUE(acc2.FindVertex(pk, View::NEW).has_value()); EXPECT_EQ(CountVertices(acc2, View::NEW), 1U); - ASSERT_FALSE(acc2.Commit().HasError()); + acc2.Commit(GetNextHlc()); } - auto acc3 = store.Access(); // read transaction - auto acc4 = store.Access(); // write transaction (aborted) + auto acc3 = store.Access(GetNextHlc()); // read transaction + auto acc4 = store.Access(GetNextHlc()); // write transaction (aborted) // Check whether the vertex exists in transaction 1 ASSERT_FALSE(acc1.FindVertex(pk, View::OLD).has_value()); @@ -383,8 +458,8 @@ TEST_F(StorageV3, VertexDeleteAbort) { acc4.Abort(); } - auto acc5 = store.Access(); // read transaction - auto acc6 = store.Access(); // write transaction + auto acc5 = store.Access(GetNextHlc()); // read transaction + auto acc6 = store.Access(GetNextHlc()); // write transaction // Check whether the vertex exists in transaction 1 ASSERT_FALSE(acc1.FindVertex(pk, View::OLD).has_value()); @@ -420,10 +495,10 @@ TEST_F(StorageV3, VertexDeleteAbort) { EXPECT_EQ(CountVertices(acc6, View::OLD), 0U); EXPECT_EQ(CountVertices(acc6, View::NEW), 0U); - ASSERT_FALSE(acc6.Commit().HasError()); + acc6.Commit(GetNextHlc()); } - auto acc7 = store.Access(); // read transaction + auto acc7 = store.Access(GetNextHlc()); // read transaction // Check whether the vertex exists in transaction 1 ASSERT_FALSE(acc1.FindVertex(pk, View::OLD).has_value()); @@ -450,23 +525,23 @@ TEST_F(StorageV3, VertexDeleteAbort) { EXPECT_EQ(CountVertices(acc7, View::NEW), 0U); // Commit all accessors - ASSERT_FALSE(acc1.Commit().HasError()); - ASSERT_FALSE(acc3.Commit().HasError()); - ASSERT_FALSE(acc5.Commit().HasError()); - ASSERT_FALSE(acc7.Commit().HasError()); + acc1.Commit(GetNextHlc()); + acc3.Commit(GetNextHlc()); + acc5.Commit(GetNextHlc()); + acc7.Commit(GetNextHlc()); } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexDeleteSerializationError) { +TEST_P(StorageV3, VertexDeleteSerializationError) { // Create vertex { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); // Delete vertex in accessor 1 { @@ -513,29 +588,30 @@ TEST_F(StorageV3, VertexDeleteSerializationError) { } // Finalize both accessors - ASSERT_FALSE(acc1.Commit().HasError()); + acc1.Commit(GetNextHlc()); acc2.Abort(); + CleanupHlc(last_hlc); // Check whether the vertex exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_FALSE(vertex); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); EXPECT_EQ(CountVertices(acc, View::NEW), 0U); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexDeleteSpecialCases) { +TEST_P(StorageV3, VertexDeleteSpecialCases) { std::vector pk1{PropertyValue{0}}; std::vector pk2{PropertyValue(2)}; // Create vertex and delete it in the same transaction, but abort the // transaction { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk1, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); @@ -554,7 +630,7 @@ TEST_F(StorageV3, VertexDeleteSpecialCases) { // Create vertex and delete it in the same transaction { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}}); ASSERT_FALSE(acc.FindVertex(pk2, View::OLD).has_value()); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); @@ -568,12 +644,12 @@ TEST_F(StorageV3, VertexDeleteSpecialCases) { acc.AdvanceCommand(); EXPECT_EQ(CountVertices(acc, View::OLD), 0U); EXPECT_EQ(CountVertices(acc, View::NEW), 0U); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the vertices exist { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); ASSERT_FALSE(acc.FindVertex(pk1, View::OLD).has_value()); ASSERT_FALSE(acc.FindVertex(pk1, View::NEW).has_value()); ASSERT_FALSE(acc.FindVertex(pk2, View::OLD).has_value()); @@ -593,19 +669,19 @@ void AssertErrorInVariant(TResultHolder &holder, TError error_type) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexDeleteLabel) { +TEST_P(StorageV3, VertexDeleteLabel) { // Create the vertex { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Add label, delete the vertex and check the label API (same command) { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::NEW); ASSERT_TRUE(vertex); @@ -656,7 +732,7 @@ TEST_F(StorageV3, VertexDeleteLabel) { // Add label, delete the vertex and check the label API (different command) { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::NEW); ASSERT_TRUE(vertex); @@ -737,19 +813,19 @@ TEST_F(StorageV3, VertexDeleteLabel) { } // NOLINTNEXTLINE(hicpp - special - member - functions) -TEST_F(StorageV3, VertexDeleteProperty) { +TEST_P(StorageV3, VertexDeleteProperty) { // Create the vertex { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(acc.FindVertex(pk, View::OLD).has_value()); ASSERT_TRUE(acc.FindVertex(pk, View::NEW).has_value()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Set property, delete the vertex and check the property API (same command) { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::NEW); ASSERT_TRUE(vertex); @@ -795,7 +871,7 @@ TEST_F(StorageV3, VertexDeleteProperty) { // Set property, delete the vertex and check the property API (different // command) { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::NEW); ASSERT_TRUE(vertex); @@ -870,10 +946,10 @@ TEST_F(StorageV3, VertexDeleteProperty) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexLabelCommit) { +TEST_P(StorageV3, VertexLabelCommit) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "label5"}, {4, "other"}}); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); auto label = NameToLabelId("label5"); @@ -900,10 +976,10 @@ TEST_F(StorageV3, VertexLabelCommit) { ASSERT_FALSE(res.GetValue()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -931,7 +1007,7 @@ TEST_F(StorageV3, VertexLabelCommit) { acc.Abort(); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -959,10 +1035,10 @@ TEST_F(StorageV3, VertexLabelCommit) { ASSERT_FALSE(res.GetValue()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -983,18 +1059,18 @@ TEST_F(StorageV3, VertexLabelCommit) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexLabelAbort) { +TEST_P(StorageV3, VertexLabelAbort) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "label5"}, {4, "other"}}); // Create the vertex. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Add label 5, but abort the transaction. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1027,7 +1103,7 @@ TEST_F(StorageV3, VertexLabelAbort) { // Check that label 5 doesn't exist. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1048,7 +1124,7 @@ TEST_F(StorageV3, VertexLabelAbort) { // Add label 5. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1076,12 +1152,12 @@ TEST_F(StorageV3, VertexLabelAbort) { ASSERT_FALSE(res.GetValue()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check that label 5 exists. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1111,7 +1187,7 @@ TEST_F(StorageV3, VertexLabelAbort) { // Remove label 5, but abort the transaction. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1144,7 +1220,7 @@ TEST_F(StorageV3, VertexLabelAbort) { // Check that label 5 exists. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1174,7 +1250,7 @@ TEST_F(StorageV3, VertexLabelAbort) { // Remove label 5. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1202,12 +1278,12 @@ TEST_F(StorageV3, VertexLabelAbort) { ASSERT_FALSE(res.GetValue()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check that label 5 doesn't exist. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1228,16 +1304,16 @@ TEST_F(StorageV3, VertexLabelAbort) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexLabelSerializationError) { +TEST_P(StorageV3, VertexLabelSerializationError) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "label1"}, {4, "label2"}}); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); // Add label 1 in accessor 1. { @@ -1300,12 +1376,12 @@ TEST_F(StorageV3, VertexLabelSerializationError) { } // Finalize both accessors. - ASSERT_FALSE(acc1.Commit().HasError()); + acc1.Commit(GetNextHlc()); acc2.Abort(); // Check which labels exist. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1333,10 +1409,10 @@ TEST_F(StorageV3, VertexLabelSerializationError) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexPropertyCommit) { +TEST_P(StorageV3, VertexPropertyCommit) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "property5"}, {4, "other"}}); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); auto property = NameToPropertyId("property5"); @@ -1370,10 +1446,10 @@ TEST_F(StorageV3, VertexPropertyCommit) { ASSERT_EQ(properties[property].ValueString(), "nandare"); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1401,7 +1477,7 @@ TEST_F(StorageV3, VertexPropertyCommit) { acc.Abort(); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1429,10 +1505,10 @@ TEST_F(StorageV3, VertexPropertyCommit) { ASSERT_TRUE(old_value->IsNull()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1453,18 +1529,18 @@ TEST_F(StorageV3, VertexPropertyCommit) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexPropertyAbort) { +TEST_P(StorageV3, VertexPropertyAbort) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "property5"}, {4, "other"}}); // Create the vertex. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Set property 5 to "nandare", but abort the transaction. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1504,7 +1580,7 @@ TEST_F(StorageV3, VertexPropertyAbort) { // Check that property 5 is null. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1525,7 +1601,7 @@ TEST_F(StorageV3, VertexPropertyAbort) { // Set property 5 to "nandare". { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1560,12 +1636,12 @@ TEST_F(StorageV3, VertexPropertyAbort) { ASSERT_EQ(properties[property].ValueString(), "nandare"); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check that property 5 is "nandare". { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1595,7 +1671,7 @@ TEST_F(StorageV3, VertexPropertyAbort) { // Set property 5 to null, but abort the transaction. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1636,7 +1712,7 @@ TEST_F(StorageV3, VertexPropertyAbort) { // Check that property 5 is "nandare". { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1666,7 +1742,7 @@ TEST_F(StorageV3, VertexPropertyAbort) { // Set property 5 to null. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1702,12 +1778,12 @@ TEST_F(StorageV3, VertexPropertyAbort) { ASSERT_TRUE(vertex->GetProperty(property, View::NEW)->IsNull()); ASSERT_EQ(vertex->Properties(View::NEW)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check that property 5 is null. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1728,16 +1804,16 @@ TEST_F(StorageV3, VertexPropertyAbort) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexPropertySerializationError) { +TEST_P(StorageV3, VertexPropertySerializationError) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "property1"}, {4, "property2"}}); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); // Set property 1 to 123 in accessor 1. { @@ -1794,12 +1870,12 @@ TEST_F(StorageV3, VertexPropertySerializationError) { } // Finalize both accessors. - ASSERT_FALSE(acc1.Commit().HasError()); + acc1.Commit(GetNextHlc()); acc2.Abort(); // Check which properties exist. { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -1827,9 +1903,9 @@ TEST_F(StorageV3, VertexPropertySerializationError) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, VertexLabelPropertyMixed) { +TEST_P(StorageV3, VertexLabelPropertyMixed) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "label5"}, {4, "property5"}}); - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); auto label = NameToLabelId("label5"); @@ -2063,25 +2139,25 @@ TEST_F(StorageV3, VertexLabelPropertyMixed) { ASSERT_EQ(vertex.Properties(View::OLD)->size(), 0); ASSERT_EQ(vertex.Properties(View::NEW)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } -TEST_F(StorageV3, VertexPropertyClear) { +TEST_P(StorageV3, VertexPropertyClear) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "property1"}, {4, "property2"}}); auto property1 = NameToPropertyId("property1"); auto property2 = NameToPropertyId("property2"); { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); auto old_value = vertex.SetPropertyAndValidate(property1, PropertyValue("value")); ASSERT_TRUE(old_value.HasValue()); ASSERT_TRUE(old_value->IsNull()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -2113,7 +2189,7 @@ TEST_F(StorageV3, VertexPropertyClear) { acc.Abort(); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -2121,10 +2197,10 @@ TEST_F(StorageV3, VertexPropertyClear) { ASSERT_TRUE(old_value.HasValue()); ASSERT_TRUE(old_value->IsNull()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -2154,10 +2230,10 @@ TEST_F(StorageV3, VertexPropertyClear) { ASSERT_TRUE(vertex->GetProperty(property2, View::NEW)->IsNull()); ASSERT_EQ(vertex->Properties(View::NEW).GetValue().size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -2169,13 +2245,13 @@ TEST_F(StorageV3, VertexPropertyClear) { } } -TEST_F(StorageV3, VertexNonexistentLabelPropertyEdgeAPI) { +TEST_P(StorageV3, VertexNonexistentLabelPropertyEdgeAPI) { store.StoreMapping({{1, "label"}, {2, "property"}, {3, "label1"}, {4, "property1"}, {5, "edge"}}); auto label1 = NameToLabelId("label1"); auto property1 = NameToPropertyId("property1"); - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); // Check state before (OLD view). @@ -2225,12 +2301,12 @@ TEST_F(StorageV3, VertexNonexistentLabelPropertyEdgeAPI) { ASSERT_EQ(*vertex.InDegree(View::NEW), 1); ASSERT_EQ(*vertex.OutDegree(View::NEW), 1); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } -TEST_F(StorageV3, VertexVisibilitySingleTransaction) { - auto acc1 = store.Access(); - auto acc2 = store.Access(); +TEST_P(StorageV3, VertexVisibilitySingleTransaction) { + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc1, primary_label, {}, {{primary_property, PropertyValue{0}}}); @@ -2248,7 +2324,7 @@ TEST_F(StorageV3, VertexVisibilitySingleTransaction) { ASSERT_TRUE(vertex.SetPropertyAndValidate(NameToPropertyId("meaning"), PropertyValue(42)).HasValue()); - auto acc3 = store.Access(); + auto acc3 = store.Access(GetNextHlc()); EXPECT_FALSE(acc1.FindVertex(pk, View::OLD)); EXPECT_TRUE(acc1.FindVertex(pk, View::NEW)); @@ -2281,10 +2357,10 @@ TEST_F(StorageV3, VertexVisibilitySingleTransaction) { acc3.Abort(); } -TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { +TEST_P(StorageV3, VertexVisibilityMultipleTransactions) { { - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); CreateVertexAndValidate(acc1, primary_label, {}, {{primary_property, PropertyValue{0}}}); @@ -2307,13 +2383,13 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { EXPECT_FALSE(acc2.FindVertex(pk, View::OLD)); EXPECT_FALSE(acc2.FindVertex(pk, View::NEW)); - ASSERT_FALSE(acc1.Commit().HasError()); - ASSERT_FALSE(acc2.Commit().HasError()); + acc1.Commit(GetNextHlc()); + acc2.Commit(GetNextHlc()); } { - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); auto vertex = acc1.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); @@ -2346,7 +2422,7 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { ASSERT_TRUE(vertex->SetPropertyAndValidate(NameToPropertyId("meaning"), PropertyValue(42)).HasValue()); - auto acc3 = store.Access(); + auto acc3 = store.Access(GetNextHlc()); EXPECT_TRUE(acc1.FindVertex(pk, View::OLD)); EXPECT_TRUE(acc1.FindVertex(pk, View::NEW)); @@ -2382,21 +2458,22 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { EXPECT_TRUE(acc3.FindVertex(pk, View::OLD)); EXPECT_TRUE(acc3.FindVertex(pk, View::NEW)); - ASSERT_FALSE(acc1.Commit().HasError()); - ASSERT_FALSE(acc2.Commit().HasError()); - ASSERT_FALSE(acc3.Commit().HasError()); + acc1.Commit(GetNextHlc()); + acc2.Commit(GetNextHlc()); + acc3.Commit(GetNextHlc()); + CleanupHlc(last_hlc); } { - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); auto vertex = acc1.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); ASSERT_TRUE(acc1.DeleteVertex(&*vertex).HasValue()); - auto acc3 = store.Access(); + auto acc3 = store.Access(GetNextHlc()); EXPECT_TRUE(acc1.FindVertex(pk, View::OLD)); EXPECT_FALSE(acc1.FindVertex(pk, View::NEW)); @@ -2435,10 +2512,11 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { acc1.Abort(); acc2.Abort(); acc3.Abort(); + CleanupHlc(last_hlc); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); EXPECT_TRUE(acc.FindVertex(pk, View::OLD)); EXPECT_TRUE(acc.FindVertex(pk, View::NEW)); @@ -2452,15 +2530,15 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { } { - auto acc1 = store.Access(); - auto acc2 = store.Access(); + auto acc1 = store.Access(GetNextHlc()); + auto acc2 = store.Access(GetNextHlc()); auto vertex = acc1.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); ASSERT_TRUE(acc1.DeleteVertex(&*vertex).HasValue()); - auto acc3 = store.Access(); + auto acc3 = store.Access(GetNextHlc()); EXPECT_TRUE(acc1.FindVertex(pk, View::OLD)); EXPECT_FALSE(acc1.FindVertex(pk, View::NEW)); @@ -2496,13 +2574,13 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { EXPECT_TRUE(acc3.FindVertex(pk, View::OLD)); EXPECT_TRUE(acc3.FindVertex(pk, View::NEW)); - ASSERT_FALSE(acc1.Commit().HasError()); - ASSERT_FALSE(acc2.Commit().HasError()); - ASSERT_FALSE(acc3.Commit().HasError()); + acc1.Commit(GetNextHlc()); + acc2.Commit(GetNextHlc()); + acc3.Commit(GetNextHlc()); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); EXPECT_FALSE(acc.FindVertex(pk, View::OLD)); EXPECT_FALSE(acc.FindVertex(pk, View::NEW)); @@ -2517,19 +2595,19 @@ TEST_F(StorageV3, VertexVisibilityMultipleTransactions) { } // NOLINTNEXTLINE(hicpp-special-member-functions) -TEST_F(StorageV3, DeletedVertexAccessor) { +TEST_P(StorageV3, DeletedVertexAccessor) { const auto property1 = NameToPropertyId("property1"); const PropertyValue property_value{"property_value"}; // Create the vertex { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue{0}}}); ASSERT_FALSE(vertex.SetPropertyAndValidate(property1, property_value).HasError()); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.FindVertex(pk, View::OLD); ASSERT_TRUE(vertex); auto maybe_deleted_vertex = acc.DeleteVertex(&*vertex); @@ -2544,7 +2622,7 @@ TEST_F(StorageV3, DeletedVertexAccessor) { const auto maybe_property = deleted_vertex->GetProperty(property1, View::OLD); ASSERT_FALSE(maybe_property.HasError()); ASSERT_EQ(property_value, *maybe_property); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); { // you can call read only methods and get valid results even after the @@ -2556,9 +2634,9 @@ TEST_F(StorageV3, DeletedVertexAccessor) { } } -TEST_F(StorageV3, TestCreateVertexAndValidate) { +TEST_P(StorageV3, TestCreateVertexAndValidate) { { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); const auto prop1 = NameToPropertyId("prop1"); auto vertex = acc.CreateVertexAndValidate(primary_label, {label1}, @@ -2573,7 +2651,7 @@ TEST_F(StorageV3, TestCreateVertexAndValidate) { (std::map{{prop1, PropertyValue(111)}})); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); const auto prop1 = NameToPropertyId("prop1"); EXPECT_THROW( @@ -2589,14 +2667,14 @@ TEST_F(StorageV3, TestCreateVertexAndValidate) { Shard store(primary_label, pk, std::nullopt); ASSERT_TRUE(store.CreateSchema(primary_label, {storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}})); - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex1 = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(0)}}); auto vertex2 = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(0)}}); }, ""); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.CreateVertexAndValidate(primary_label, {primary_label}, {{primary_property, PropertyValue(0)}}); ASSERT_TRUE(vertex.HasError()); ASSERT_TRUE(std::holds_alternative(vertex.GetError())); @@ -2604,7 +2682,7 @@ TEST_F(StorageV3, TestCreateVertexAndValidate) { SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, primary_label)); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.CreateVertexAndValidate(primary_label, {primary_label}, {{primary_property, PropertyValue(0)}}); ASSERT_TRUE(vertex.HasError()); ASSERT_TRUE(std::holds_alternative(vertex.GetError())); @@ -2612,7 +2690,7 @@ TEST_F(StorageV3, TestCreateVertexAndValidate) { SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, primary_label)); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.CreateVertexAndValidate(primary_label, {}, {}); ASSERT_TRUE(vertex.HasError()); ASSERT_TRUE(std::holds_alternative(vertex.GetError())); @@ -2621,7 +2699,7 @@ TEST_F(StorageV3, TestCreateVertexAndValidate) { {primary_property, common::SchemaType::INT})); } { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue("test")}}); ASSERT_TRUE(vertex.HasError()); ASSERT_TRUE(std::holds_alternative(vertex.GetError())); diff --git a/tests/unit/storage_v3_edge.cpp b/tests/unit/storage_v3_edge.cpp index 807207844..fdec80fab 100644 --- a/tests/unit/storage_v3_edge.cpp +++ b/tests/unit/storage_v3_edge.cpp @@ -44,6 +44,12 @@ class StorageEdgeTest : public ::testing::TestWithParam { return acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, key}}); } + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += std::chrono::seconds(10); + return last_hlc; + } + static constexpr int64_t min_primary_key_value{0}; static constexpr int64_t max_primary_key_value{10000}; const LabelId primary_label{LabelId::FromUint(1)}; @@ -52,6 +58,7 @@ class StorageEdgeTest : public ::testing::TestWithParam { std::vector{PropertyValue{max_primary_key_value}}, Config{.items = {.properties_on_edges = GetParam()}}}; const PropertyId primary_property{PropertyId::FromUint(2)}; + coordinator::Hlc last_hlc{0, io::Time{}}; }; INSTANTIATE_TEST_CASE_P(EdgesWithProperties, StorageEdgeTest, ::testing::Values(true)); @@ -65,7 +72,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { const PropertyValue non_existing_key{2}; const auto et = NameToEdgeTypeId("et5"); const auto edge_id = Gid::FromUint(0U); - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto [from_id, to_id] = std::invoke([this, &from_key, &to_key, &acc]() mutable -> std::pair { auto from_id = CreateVertex(acc, from_key)->Id(View::NEW).GetValue(); @@ -77,11 +84,11 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { const VertexId to_id_with_different_label{NameToLabelId("different_label"), to_id.primary_key}; const VertexId non_existing_id{primary_label, {non_existing_key}}; - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); // Create edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex({to_id.primary_key}, View::NEW); ASSERT_TRUE(vertex_from); @@ -157,12 +164,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_to->InEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from.has_value()); @@ -269,7 +276,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { ASSERT_EQ(vertex_to->InEdges(View::OLD, {et, other_et}, &from_id_with_different_label)->size(), 0); ASSERT_EQ(vertex_to->InEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } } @@ -281,17 +288,17 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // // Create vertices // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_to = acc.CreateVertex(); // auto vertex_from = acc.CreateVertex(); // gid_to = vertex_to.Gid(); // gid_from = vertex_from.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -354,12 +361,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -441,7 +448,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -452,15 +459,15 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // // Create vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid_vertex = vertex.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -513,12 +520,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex->InEdges(View::NEW, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->InEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -586,7 +593,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex->OutEdges(View::NEW, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->OutEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -598,17 +605,17 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // // Create vertices // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.CreateVertex(); // auto vertex_to = acc.CreateVertex(); // gid_from = vertex_from.Gid(); // gid_to = vertex_to.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -676,7 +683,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -700,12 +707,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -768,12 +775,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -855,7 +862,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -865,7 +872,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { const PropertyValue from_key{0}; const PropertyValue to_key{1}; const PropertyValue non_existing_key{2}; - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto [from_id, to_id] = std::invoke([this, &from_key, &to_key, &acc]() mutable -> std::pair { auto from_id = CreateVertex(acc, from_key)->Id(View::NEW).GetValue(); @@ -879,11 +886,11 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { const VertexId to_id_with_different_label{NameToLabelId("different_label"), to_id.primary_key}; const VertexId non_existing_id{primary_label, {non_existing_key}}; - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); // Create edge but abort { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex({to_id.primary_key}, View::NEW); ASSERT_TRUE(vertex_from); @@ -966,7 +973,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -990,12 +997,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Create edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -1073,12 +1080,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { ASSERT_EQ(vertex_to->OutEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_to->OutEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -1165,7 +1172,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { ASSERT_EQ(vertex_to->OutEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_to->OutEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } } @@ -1176,15 +1183,15 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // // Create vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid_vertex = vertex.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1242,7 +1249,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1256,12 +1263,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1314,12 +1321,12 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // ASSERT_EQ(vertex->InEdges(View::NEW, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->InEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1391,7 +1398,7 @@ TEST_P(StorageEdgeTest, EdgeCreateFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); // ASSERT_EQ(vertex->OutEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -1401,7 +1408,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { const PropertyValue from_key{0}; const PropertyValue to_key{max_primary_key_value}; const PropertyValue non_existing_key{2}; - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto from_id = std::invoke( [this, &from_key, &acc]() mutable -> VertexId { return CreateVertex(acc, from_key)->Id(View::NEW).GetValue(); }); const VertexId to_id{primary_label, {to_key}}; @@ -1411,11 +1418,11 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { const VertexId to_id_with_different_label{NameToLabelId("different_label"), to_id.primary_key}; const VertexId non_existing_id{primary_label, {non_existing_key}}; - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); // Create edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); ASSERT_FALSE(acc.FindVertex(to_id.primary_key, View::NEW).has_value()); @@ -1463,12 +1470,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { ASSERT_EQ(vertex_from->OutEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_from->OutEdges(View::NEW, {et, other_et}, &to_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -1515,12 +1522,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { ASSERT_EQ(vertex_from->OutEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_from->OutEdges(View::NEW, {et, other_et}, &to_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Delete edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -1562,12 +1569,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { ASSERT_EQ(vertex_from->OutEdges(View::OLD, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_from->OutEdges(View::OLD, {et, other_et}, &to_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -1581,7 +1588,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { ASSERT_EQ(vertex_from->OutEdges(View::NEW)->size(), 0); ASSERT_EQ(*vertex_from->OutDegree(View::NEW), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } } @@ -1593,17 +1600,17 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // // Create vertices // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_to = acc.CreateVertex(); // auto vertex_from = acc.CreateVertex(); // gid_from = vertex_from.Gid(); // gid_to = vertex_to.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -1666,12 +1673,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -1753,12 +1760,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -1820,12 +1827,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -1849,7 +1856,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -1860,15 +1867,15 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // // Create vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid_vertex = vertex.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1921,12 +1928,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex->InEdges(View::NEW, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->InEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -1998,12 +2005,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); // ASSERT_EQ(vertex->OutEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -2055,12 +2062,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -2074,7 +2081,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -2086,17 +2093,17 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // // Create vertices // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.CreateVertex(); // auto vertex_to = acc.CreateVertex(); // gid_from = vertex_from.Gid(); // gid_to = vertex_to.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2159,12 +2166,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2246,12 +2253,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete the edge, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2318,7 +2325,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2400,12 +2407,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); // ASSERT_EQ(vertex_to->InEdges(View::NEW, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete the edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2467,12 +2474,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &from_id)->size(), 1); // ASSERT_EQ(vertex_to->InEdges(View::OLD, {}, &to_id)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -2496,7 +2503,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromSmallerCommit) { // ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -2506,7 +2513,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { const PropertyValue from_key{max_primary_key_value}; const PropertyValue to_key{0}; const PropertyValue non_existing_key{2}; - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); const auto to_id = std::invoke( [this, &to_key, &acc]() mutable -> VertexId { return CreateVertex(acc, to_key)->Id(View::NEW).GetValue(); }); const VertexId from_id{primary_label, {from_key}}; @@ -2516,10 +2523,10 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { const VertexId from_id_with_different_label{NameToLabelId("different_label"), from_id.primary_key}; const VertexId non_existing_id{primary_label, {non_existing_key}}; - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); // Create edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_to); @@ -2563,12 +2570,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { ASSERT_EQ(vertex_to->OutEdges(View::NEW, {}, &non_existing_id)->size(), 0); ASSERT_EQ(vertex_to->OutEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); // Check edges without filters @@ -2623,12 +2630,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { ASSERT_EQ(vertex_to->InEdges(View::OLD, {et, other_et}, &from_id_with_different_label)->size(), 0); ASSERT_EQ(vertex_to->InEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Delete the edge, but abort the transaction { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_to); @@ -2674,7 +2681,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_to); @@ -2730,12 +2737,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { ASSERT_EQ(vertex_to->InEdges(View::OLD, {et, other_et}, &from_id_with_different_label)->size(), 0); ASSERT_EQ(vertex_to->InEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Delete the edge { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_to); @@ -2786,12 +2793,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { ASSERT_EQ(vertex_to->InEdges(View::OLD, {et, other_et}, &from_id_with_different_label)->size(), 0); ASSERT_EQ(vertex_to->InEdges(View::NEW, {et, other_et}, &from_id_with_different_label)->size(), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check whether the edge exists { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_to); @@ -2805,7 +2812,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } } @@ -2816,15 +2823,15 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // // Create vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid_vertex = vertex.Gid(); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Create edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -2877,12 +2884,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // ASSERT_EQ(vertex->InEdges(View::NEW, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->InEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -2954,12 +2961,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); // ASSERT_EQ(vertex->OutEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete the edge, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -3016,7 +3023,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -3088,12 +3095,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); // ASSERT_EQ(vertex->OutEdges(View::NEW, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Delete the edge // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -3145,12 +3152,12 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::OLD, {}, &*vertex)->size(), 1); // ASSERT_EQ(vertex->OutEdges(View::OLD, {other_et}, &*vertex)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check whether the edge exists // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid_vertex, View::NEW); // ASSERT_TRUE(vertex); @@ -3164,7 +3171,7 @@ TEST_P(StorageEdgeTest, EdgeDeleteFromLargerAbort) { // ASSERT_EQ(vertex->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // } @@ -3176,14 +3183,13 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { const PropertyValue non_existing_key{2}; const auto et = NameToEdgeTypeId("et5"); const auto edge_id = Gid::FromUint(0U); - auto acc = store.Access(); const VertexId from_id{primary_label, {from_key}}; const VertexId to_id{primary_label, {to_key}}; const VertexId non_existing_id{primary_label, {non_existing_key}}; // Create dataset { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = CreateVertex(acc, from_key).GetValue(); auto vertex_to = CreateVertex(acc, to_key).GetValue(); @@ -3224,12 +3230,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { ASSERT_EQ(vertex_to.OutEdges(View::NEW)->size(), 0); ASSERT_EQ(*vertex_to.OutDegree(View::NEW), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Detach delete vertex { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_TRUE(vertex_from); @@ -3287,12 +3293,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Check dataset { - auto acc = store.Access(); + auto acc = store.Access(GetNextHlc()); auto vertex_from = acc.FindVertex(from_id.primary_key, View::NEW); auto vertex_to = acc.FindVertex(to_id.primary_key, View::NEW); ASSERT_FALSE(vertex_from); @@ -3318,7 +3324,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Create dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.CreateVertex(); // auto vertex2 = acc.CreateVertex(); @@ -3436,12 +3442,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // } // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Detach delete vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_TRUE(vertex1); @@ -3570,12 +3576,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(e.ToVertex(), *vertex2); // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_FALSE(vertex1); @@ -3639,7 +3645,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Create dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.CreateVertex(); // auto vertex_to = acc.CreateVertex(); @@ -3683,12 +3689,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(vertex_to.OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to.OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Detach delete vertex, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -3751,7 +3757,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Check dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -3787,12 +3793,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Detach delete vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_TRUE(vertex_from); @@ -3850,12 +3856,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(vertex_to->OutEdges(View::NEW)->size(), 0); // ASSERT_EQ(*vertex_to->OutDegree(View::NEW), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex_from = acc.FindVertex(gid_from, View::NEW); // auto vertex_to = acc.FindVertex(gid_to, View::NEW); // ASSERT_FALSE(vertex_from); @@ -3881,7 +3887,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Create dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.CreateVertex(); // auto vertex2 = acc.CreateVertex(); @@ -3999,12 +4005,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // } // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Detach delete vertex, but abort the transaction // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_TRUE(vertex1); @@ -4138,7 +4144,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Check dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_TRUE(vertex1); @@ -4303,12 +4309,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // } // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Detach delete vertex // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_TRUE(vertex1); @@ -4437,12 +4443,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(e.ToVertex(), *vertex2); // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check dataset // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex1 = acc.FindVertex(gid_vertex1, View::NEW); // auto vertex2 = acc.FindVertex(gid_vertex2, View::NEW); // ASSERT_FALSE(vertex1); @@ -4503,7 +4509,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // memgraph::storage::Storage store({.items = {.properties_on_edges = true}}); // memgraph::storage::Gid gid = memgraph::storage::Gid::FromUint(std::numeric_limits::max()); // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -4543,10 +4549,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(properties[property].ValueString(), "nandare"); // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4575,7 +4581,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // acc.Abort(); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4604,10 +4610,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_TRUE(old_value->IsNull()); // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4635,7 +4641,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Create the vertex. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -4643,12 +4649,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(edge.EdgeType(), et); // ASSERT_EQ(edge.FromVertex(), vertex); // ASSERT_EQ(edge.ToVertex(), vertex); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Set property 5 to "nandare", but abort the transaction. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4689,7 +4695,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Check that property 5 is null. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4711,7 +4717,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Set property 5 to "nandare". // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4747,12 +4753,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(properties[property].ValueString(), "nandare"); // } -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check that property 5 is "nandare". // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4783,7 +4789,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Set property 5 to null, but abort the transaction. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4825,7 +4831,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Check that property 5 is "nandare". // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4856,7 +4862,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // // Set property 5 to null. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4893,12 +4899,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_TRUE(edge.GetProperty(property, View::NEW)->IsNull()); // ASSERT_EQ(edge.Properties(View::NEW)->size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // // Check that property 5 is null. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -4924,7 +4930,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // memgraph::storage::Storage store({.items = {.properties_on_edges = true}}); // memgraph::storage::Gid gid = memgraph::storage::Gid::FromUint(std::numeric_limits::max()); // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -4932,11 +4938,11 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(edge.EdgeType(), et); // ASSERT_EQ(edge.FromVertex(), vertex); // ASSERT_EQ(edge.ToVertex(), vertex); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } -// auto acc1 = store.Access(); -// auto acc2 = store.Access(); +// auto acc1 = store.Access(GetNextHlc()); +// auto acc2 = store.Access(GetNextHlc()); // // Set property 1 to 123 in accessor 1. // { @@ -4996,12 +5002,12 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // } // // Finalize both accessors. -// ASSERT_FALSE(acc1.Commit().HasError()); +// ASSERT_FALSE(acc1.Commit(GetNextHlc()); // acc2.Abort(); // // Check which properties exist. // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5035,7 +5041,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // auto property1 = store.NameToProperty("property1"); // auto property2 = store.NameToProperty("property2"); // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -5048,10 +5054,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_TRUE(old_value.HasValue()); // ASSERT_TRUE(old_value->IsNull()); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5084,7 +5090,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // acc.Abort(); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5093,10 +5099,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_TRUE(old_value.HasValue()); // ASSERT_TRUE(old_value->IsNull()); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5127,10 +5133,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_TRUE(edge.GetProperty(property2, View::NEW)->IsNull()); // ASSERT_EQ(edge.Properties(View::NEW).GetValue().size(), 0); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5148,7 +5154,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // memgraph::storage::Storage store({.items = {.properties_on_edges = false}}); // memgraph::storage::Gid gid = memgraph::storage::Gid::FromUint(std::numeric_limits::max()); // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -5156,10 +5162,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(edge.EdgeType(), et); // ASSERT_EQ(edge.FromVertex(), vertex); // ASSERT_EQ(edge.ToVertex(), vertex); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5190,7 +5196,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // acc.Abort(); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5216,7 +5222,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // memgraph::storage::Storage store({.items = {.properties_on_edges = false}}); // memgraph::storage::Gid gid; // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // gid = vertex.Gid(); // auto et = acc.NameToEdgeType("et5"); @@ -5224,10 +5230,10 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(edge.EdgeType(), et); // ASSERT_EQ(edge.FromVertex(), vertex); // ASSERT_EQ(edge.ToVertex(), vertex); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } // { -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.FindVertex(gid, View::OLD); // ASSERT_TRUE(vertex); // auto edge = vertex->OutEdges(View::NEW).GetValue()[0]; @@ -5243,7 +5249,7 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // auto property = store.NameToProperty("property"); -// auto acc = store.Access(); +// auto acc = store.Access(GetNextHlc()); // auto vertex = acc.CreateVertex(); // auto edge = acc.CreateEdge(&vertex, &vertex, acc.NameToEdgeType("edge")); // ASSERT_TRUE(edge.HasValue()); @@ -5267,6 +5273,6 @@ TEST_P(StorageEdgeTest, VertexDetachDeleteSingleCommit) { // ASSERT_EQ(edge->Properties(View::NEW)->size(), 1); // ASSERT_EQ(*edge->GetProperty(property, View::NEW), memgraph::storage::PropertyValue("value")); -// ASSERT_FALSE(acc.Commit().HasError()); +// acc.Commit(GetNextHlc()); // } } // namespace memgraph::storage::v3::tests diff --git a/tests/unit/storage_v3_expr.cpp b/tests/unit/storage_v3_expr.cpp index 6796fb13f..5f165a405 100644 --- a/tests/unit/storage_v3_expr.cpp +++ b/tests/unit/storage_v3_expr.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -134,7 +135,7 @@ class ExpressionEvaluatorTest : public ::testing::Test { PrimaryKey min_pk{PropertyValue(0)}; Shard db{primary_label, min_pk, std::nullopt}; - Shard::Accessor storage_dba{db.Access()}; + Shard::Accessor storage_dba{db.Access(GetNextHlc())}; DbAccessor dba{&storage_dba}; AstStorage storage; @@ -145,6 +146,8 @@ class ExpressionEvaluatorTest : public ::testing::Test { Frame frame{128}; ExpressionEvaluator eval{&frame, symbol_table, ctx, &dba, View::OLD}; + coordinator::Hlc last_hlc{0, io::Time{}}; + void SetUp() override { db.StoreMapping({{1, "label"}, {2, "property"}}); ASSERT_TRUE( @@ -186,6 +189,12 @@ class ExpressionEvaluatorTest : public ::testing::Test { "EvaluationContext for allocations!"; return value; } + + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += std::chrono::seconds(1); + return last_hlc; + } }; TEST_F(ExpressionEvaluatorTest, OrOperator) { diff --git a/tests/unit/storage_v3_indices.cpp b/tests/unit/storage_v3_indices.cpp index a344aff77..68dc3aed7 100644 --- a/tests/unit/storage_v3_indices.cpp +++ b/tests/unit/storage_v3_indices.cpp @@ -10,6 +10,7 @@ // licenses/APL.txt. #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "storage/v3/property_value.hpp" #include "storage/v3/storage.hpp" #include "storage/v3/temporal.hpp" +#include "storage/v3/view.hpp" // NOLINTNEXTLINE(google-build-using-namespace) @@ -44,19 +46,23 @@ class IndexTest : public testing::Test { Shard storage{primary_label, pk, std::nullopt}; const PropertyId primary_property{PropertyId::FromUint(2)}; - const LabelId label1{LabelId::FromUint(3)}; - const LabelId label2{LabelId::FromUint(4)}; const PropertyId prop_id{PropertyId::FromUint(5)}; const PropertyId prop_val{PropertyId::FromUint(6)}; + const LabelId label1{LabelId::FromUint(3)}; + const LabelId label2{LabelId::FromUint(4)}; + static constexpr std::chrono::seconds wall_clock_increment{10}; + static constexpr std::chrono::seconds reclamation_interval{wall_clock_increment / 2}; + static constexpr io::Duration one_time_unit{1}; int primary_key_id{0}; int vertex_id{0}; + coordinator::Hlc last_hlc{0, io::Time{}}; LabelId NameToLabelId(std::string_view label_name) { return storage.NameToLabel(label_name); } PropertyId NameToPropertyId(std::string_view property_name) { return storage.NameToProperty(property_name); } - VertexAccessor CreateVertex(Shard::Accessor *accessor) { - auto vertex = *accessor->CreateVertexAndValidate( + VertexAccessor CreateVertex(Shard::Accessor &accessor) { + auto vertex = *accessor.CreateVertexAndValidate( primary_label, {}, {{primary_property, PropertyValue(primary_key_id++)}, {prop_id, PropertyValue(vertex_id++)}}); return vertex; @@ -82,37 +88,47 @@ class IndexTest : public testing::Test { } return ret; } + + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += wall_clock_increment; + return last_hlc; + } + + void CleanupHlc(const coordinator::Hlc hlc) { + storage.CollectGarbage(hlc.coordinator_wall_clock + reclamation_interval + one_time_unit); + } }; // NOLINTNEXTLINE(hicpp-special-member-functions) TEST_F(IndexTest, LabelIndexCreate) { { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelIndexExists(label1)); } EXPECT_EQ(storage.ListAllIndices().label.size(), 0); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 10; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } EXPECT_TRUE(storage.CreateIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9)); EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9)); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 10; i < 20; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } @@ -131,9 +147,9 @@ TEST_F(IndexTest, LabelIndexCreate) { } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 10; i < 20; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } @@ -148,11 +164,11 @@ TEST_F(IndexTest, LabelIndexCreate) { EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29)); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29)); EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), @@ -165,67 +181,67 @@ TEST_F(IndexTest, LabelIndexCreate) { EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29)); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } } // NOLINTNEXTLINE(hicpp-special-member-functions) TEST_F(IndexTest, LabelIndexDrop) { { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelIndexExists(label1)); } EXPECT_EQ(storage.ListAllIndices().label.size(), 0); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 10; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } EXPECT_TRUE(storage.CreateIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9)); EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9)); } EXPECT_TRUE(storage.DropIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelIndexExists(label1)); } EXPECT_EQ(storage.ListAllIndices().label.size(), 0); EXPECT_FALSE(storage.DropIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelIndexExists(label1)); } EXPECT_EQ(storage.ListAllIndices().label.size(), 0); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 10; i < 20; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } EXPECT_TRUE(storage.CreateIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(acc.LabelIndexExists(label1)); } EXPECT_THAT(storage.ListAllIndices().label, UnorderedElementsAre(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19)); @@ -253,7 +269,7 @@ TEST_F(IndexTest, LabelIndexBasic) { EXPECT_TRUE(storage.CreateIndex(label1)); EXPECT_TRUE(storage.CreateIndex(label2)); - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(storage.ListAllIndices().label, UnorderedElementsAre(label1, label2)); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), IsEmpty()); EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), IsEmpty()); @@ -261,7 +277,7 @@ TEST_F(IndexTest, LabelIndexBasic) { EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), IsEmpty()); for (int i = 0; i < 10; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); } @@ -319,19 +335,19 @@ TEST_F(IndexTest, LabelIndexDuplicateVersions) { EXPECT_TRUE(storage.CreateIndex(label2)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 5; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); } EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4)); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); for (auto vertex : acc.Vertices(View::OLD)) { @@ -355,12 +371,12 @@ TEST_F(IndexTest, LabelIndexTransactionalIsolation) { EXPECT_TRUE(storage.CreateIndex(label1)); EXPECT_TRUE(storage.CreateIndex(label2)); - auto acc_before = storage.Access(); - auto acc = storage.Access(); - auto acc_after = storage.Access(); + auto acc_before = storage.Access(GetNextHlc()); + auto acc = storage.Access(GetNextHlc()); + auto acc_after = storage.Access(GetNextHlc()); for (int i = 0; i < 5; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); } @@ -368,9 +384,9 @@ TEST_F(IndexTest, LabelIndexTransactionalIsolation) { EXPECT_THAT(GetIds(acc_before.Vertices(label1, View::NEW), View::NEW), IsEmpty()); EXPECT_THAT(GetIds(acc_after.Vertices(label1, View::NEW), View::NEW), IsEmpty()); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); - auto acc_after_commit = storage.Access(); + auto acc_after_commit = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc_before.Vertices(label1, View::NEW), View::NEW), IsEmpty()); EXPECT_THAT(GetIds(acc_after.Vertices(label1, View::NEW), View::NEW), IsEmpty()); @@ -382,9 +398,9 @@ TEST_F(IndexTest, LabelIndexCountEstimate) { EXPECT_TRUE(storage.CreateIndex(label1)); EXPECT_TRUE(storage.CreateIndex(label2)); - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 20; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 3 ? label1 : label2)); } @@ -397,12 +413,12 @@ TEST_F(IndexTest, LabelPropertyIndexCreateAndDrop) { EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0); EXPECT_TRUE(storage.CreateIndex(label1, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(acc.LabelPropertyIndexExists(label1, prop_id)); } EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(std::make_pair(label1, prop_id))); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelPropertyIndexExists(label2, prop_id)); } EXPECT_FALSE(storage.CreateIndex(label1, prop_id)); @@ -410,7 +426,7 @@ TEST_F(IndexTest, LabelPropertyIndexCreateAndDrop) { EXPECT_TRUE(storage.CreateIndex(label2, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(acc.LabelPropertyIndexExists(label2, prop_id)); } EXPECT_THAT(storage.ListAllIndices().label_property, @@ -418,7 +434,7 @@ TEST_F(IndexTest, LabelPropertyIndexCreateAndDrop) { EXPECT_TRUE(storage.DropIndex(label1, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelPropertyIndexExists(label1, prop_id)); } EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(std::make_pair(label2, prop_id))); @@ -426,7 +442,7 @@ TEST_F(IndexTest, LabelPropertyIndexCreateAndDrop) { EXPECT_TRUE(storage.DropIndex(label2, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelPropertyIndexExists(label2, prop_id)); } EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0); @@ -442,11 +458,11 @@ TEST_F(IndexTest, LabelPropertyIndexBasic) { storage.CreateIndex(label1, prop_val); storage.CreateIndex(label2, prop_val); - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); for (int i = 0; i < 10; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2)); ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i))); } @@ -501,20 +517,20 @@ TEST_F(IndexTest, LabelPropertyIndexBasic) { TEST_F(IndexTest, LabelPropertyIndexDuplicateVersions) { storage.CreateIndex(label1, prop_val); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 5; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i))); } EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4)); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); for (auto vertex : acc.Vertices(View::OLD)) { @@ -536,12 +552,12 @@ TEST_F(IndexTest, LabelPropertyIndexDuplicateVersions) { TEST_F(IndexTest, LabelPropertyIndexTransactionalIsolation) { storage.CreateIndex(label1, prop_val); - auto acc_before = storage.Access(); - auto acc = storage.Access(); - auto acc_after = storage.Access(); + auto acc_before = storage.Access(GetNextHlc()); + auto acc = storage.Access(GetNextHlc()); + auto acc_after = storage.Access(GetNextHlc()); for (int i = 0; i < 5; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i))); } @@ -550,9 +566,9 @@ TEST_F(IndexTest, LabelPropertyIndexTransactionalIsolation) { EXPECT_THAT(GetIds(acc_before.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); EXPECT_THAT(GetIds(acc_after.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); - auto acc_after_commit = storage.Access(); + auto acc_after_commit = storage.Access(GetNextHlc()); EXPECT_THAT(GetIds(acc_before.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); EXPECT_THAT(GetIds(acc_after.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); @@ -571,17 +587,17 @@ TEST_F(IndexTest, LabelPropertyIndexFiltering) { storage.CreateIndex(label1, prop_val); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 10; ++i) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, i % 2 ? PropertyValue(i / 2) : PropertyValue(i / 2.0))); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 0; i < 5; ++i) { EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, PropertyValue(i), View::OLD)), UnorderedElementsAre(2 * i, 2 * i + 1)); @@ -628,10 +644,10 @@ TEST_F(IndexTest, LabelPropertyIndexFiltering) { TEST_F(IndexTest, LabelPropertyIndexCountEstimate) { storage.CreateIndex(label1, prop_val); - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (int i = 1; i <= 10; ++i) { for (int j = 0; j < i; ++j) { - auto vertex = CreateVertex(&acc); + auto vertex = CreateVertex(acc); ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i))); } @@ -684,18 +700,18 @@ TEST_F(IndexTest, LabelPropertyIndexMixedIteration) { // Create vertices, each with one of the values above. { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); for (const auto &value : values) { auto v = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(primary_key_id++)}}); ASSERT_TRUE(v->AddLabelAndValidate(label1).HasValue()); ASSERT_TRUE(v->SetPropertyAndValidate(prop_val, value).HasValue()); } - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } // Verify that all nodes are in the index. { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); auto iterable = acc.Vertices(label1, prop_val, View::OLD); auto it = iterable.begin(); for (const auto &value : values) { @@ -712,7 +728,7 @@ TEST_F(IndexTest, LabelPropertyIndexMixedIteration) { auto verify = [&](const std::optional> &from, const std::optional> &to, const std::vector &expected) { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); auto iterable = acc.Vertices(label1, prop_val, from, to, View::OLD); size_t i = 0; for (auto it = iterable.begin(); it != iterable.end(); ++it, ++i) { @@ -866,7 +882,7 @@ TEST_F(IndexTest, LabelPropertyIndexCreateWithExistingPrimaryKey) { EXPECT_EQ(storage.ListAllIndices().label_property.size(), 1); EXPECT_EQ(storage.ListAllIndices().label.size(), 0); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(acc.LabelPropertyIndexExists(primary_label, prop_id)); } EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(Pair(primary_label, prop_id))); @@ -884,12 +900,11 @@ TEST_F(IndexTest, LabelPropertyIndexCreateWithExistingPrimaryKey) { TEST_F(IndexTest, LabelIndexCreateVertexAndValidate) { { - auto acc = storage.Access(); EXPECT_EQ(storage.ListAllIndices().label.size(), 0); EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); // Create vertices with CreateVertexAndValidate for (int i = 0; i < 5; ++i) { @@ -897,25 +912,25 @@ TEST_F(IndexTest, LabelIndexCreateVertexAndValidate) { acc.CreateVertexAndValidate(primary_label, {label1}, {{primary_property, PropertyValue(primary_key_id++)}}); ASSERT_TRUE(vertex.HasValue()); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { EXPECT_TRUE(storage.CreateIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); } } { EXPECT_TRUE(storage.DropIndex(label1)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelIndexExists(label1)); } EXPECT_EQ(storage.ListAllIndices().label.size(), 0); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(storage.CreateIndex(label1)); EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); @@ -933,12 +948,11 @@ TEST_F(IndexTest, LabelIndexCreateVertexAndValidate) { TEST_F(IndexTest, LabelPropertyIndexCreateVertexAndValidate) { { - auto acc = storage.Access(); EXPECT_EQ(storage.ListAllIndices().label.size(), 0); EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); // Create vertices with CreateVertexAndValidate for (int i = 0; i < 5; ++i) { @@ -947,12 +961,12 @@ TEST_F(IndexTest, LabelPropertyIndexCreateVertexAndValidate) { {{primary_property, PropertyValue(primary_key_id++)}, {prop_id, PropertyValue(vertex_id++)}}); ASSERT_TRUE(vertex.HasValue()); } - ASSERT_NO_ERROR(acc.Commit()); + acc.Commit(GetNextHlc()); } { EXPECT_TRUE(storage.CreateIndex(label1, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); } @@ -960,13 +974,13 @@ TEST_F(IndexTest, LabelPropertyIndexCreateVertexAndValidate) { { EXPECT_TRUE(storage.DropIndex(label1, prop_id)); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_FALSE(acc.LabelPropertyIndexExists(label1, prop_id)); } EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); EXPECT_TRUE(storage.CreateIndex(label1, prop_id)); EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4)); @@ -984,4 +998,187 @@ TEST_F(IndexTest, LabelPropertyIndexCreateVertexAndValidate) { UnorderedElementsAre(0, 1, 2, 3, 4)); } } + +TEST_F(IndexTest, CollectGarbageDeleteVertex) { + // First part + // T1 (start 1, commit 3) Creates the vertex, adds label and property + // T2 (start 2, no commit) reads nothing + // T3 (start 4, commit 6) deletes label and property + // T4 (start 5, no commit) reads the vertex + // T5 (start 7, no commit) reads nothing + + auto t1_start = GetNextHlc(); + auto t2_start = GetNextHlc(); + auto t1_commit = GetNextHlc(); + auto t3_start = GetNextHlc(); + auto t4_start = GetNextHlc(); + auto t3_commit = GetNextHlc(); + auto t5_start = GetNextHlc(); + + ASSERT_TRUE(storage.CreateIndex(label1, prop_val)); + + auto acc1 = storage.Access(t1_start); + { + auto vertex = CreateVertex(acc1); + ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); + ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(42))); + } + acc1.Commit(t1_commit); + EXPECT_THAT(GetIds(acc1.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc1.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + + auto acc2 = storage.Access(t2_start); + EXPECT_THAT(GetIds(acc2.Vertices(View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(View::NEW), View::NEW), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + + auto acc3 = storage.Access(t3_start); + { + auto vertices = acc3.Vertices(label1, prop_val, View::OLD); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + auto vertex = *vertices.begin(); + ASSERT_NO_ERROR(acc3.DeleteVertex(&vertex)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + } + acc3.Commit(t3_commit); + auto check_t3 = [this, &acc3]() { + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + }; + check_t3(); + + auto acc4 = storage.Access(t4_start); + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + + auto acc5 = storage.Access(t5_start); + auto check_t5 = [this, &acc5]() { + EXPECT_THAT(GetIds(acc5.Vertices(View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc5.Vertices(View::NEW), View::NEW), IsEmpty()); + EXPECT_THAT(GetIds(acc5.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc5.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + }; + check_t5(); + + // Second part + // Start to clean up things. + CleanupHlc(t1_start); + // As the deltas of T1 is cleaned up, T2 will see the vertex as an existing one + EXPECT_THAT(GetIds(acc2.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + check_t3(); + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + check_t5(); + + CleanupHlc(t3_start); + // As T3 got cleaned up, it will delete the vertex from the actual storage and the index + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), IsEmpty()); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + check_t5(); +} + +TEST_F(IndexTest, CollectGarbageRemoveLabel) { + // First part + // T1 (start 1, commit 3) Creates the vertex, adds label and property + // T2 (start 2, no commit) reads nothing + // T3 (start 4, commit 6) removes label + // T4 (start 5, no commit) reads the vertex + // T5 (start 7, no commit) reads nothing + + auto t1_start = GetNextHlc(); + auto t2_start = GetNextHlc(); + auto t1_commit = GetNextHlc(); + auto t3_start = GetNextHlc(); + auto t4_start = GetNextHlc(); + auto t3_commit = GetNextHlc(); + auto t5_start = GetNextHlc(); + + ASSERT_TRUE(storage.CreateIndex(label1, prop_val)); + + auto acc1 = storage.Access(t1_start); + { + auto vertex = CreateVertex(acc1); + ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1)); + ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(42))); + } + acc1.Commit(t1_commit); + EXPECT_THAT(GetIds(acc1.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc1.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + + auto acc2 = storage.Access(t2_start); + EXPECT_THAT(GetIds(acc2.Vertices(View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(View::NEW), View::NEW), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + + auto acc3 = storage.Access(t3_start); + { + auto vertices = acc3.Vertices(label1, prop_val, View::OLD); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + auto vertex = *vertices.begin(); + ASSERT_NO_ERROR(vertex.RemoveLabelAndValidate(label1)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + } + // We clean up T1 here to make sure the garbace collection doesn't remove the vertex from the indices until the vertex + // has a version that is reachable from an active transaction, even though the latest version of the vertex doesn't + // belong to the index + CleanupHlc(t1_start); + acc3.Commit(t3_commit); + auto check_t3 = [this, &acc3]() { + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc3.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + }; + check_t3(); + + auto acc4 = storage.Access(t4_start); + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + + auto acc5 = storage.Access(t5_start); + auto check_t5 = [this, &acc5]() { + EXPECT_THAT(GetIds(acc5.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc5.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc5.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc5.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + }; + check_t5(); + + // As the deltas of T1 is cleaned up, T2 will see the vertex as an existing one + EXPECT_THAT(GetIds(acc2.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc2.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + check_t3(); + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0)); + check_t5(); + + // Second part + // Clean up T3 and check the changes + CleanupHlc(t3_start); + // As T3 got cleaned up, it will delete the vertex from the index but not from the actual storage + EXPECT_THAT(GetIds(acc4.Vertices(View::OLD), View::OLD), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(View::NEW), View::NEW), UnorderedElementsAre(0)); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty()); + EXPECT_THAT(GetIds(acc4.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty()); + check_t5(); +} } // namespace memgraph::storage::v3::tests diff --git a/tests/unit/storage_v3_isolation_level.cpp b/tests/unit/storage_v3_isolation_level.cpp new file mode 100644 index 000000000..208e6884d --- /dev/null +++ b/tests/unit/storage_v3_isolation_level.cpp @@ -0,0 +1,142 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +#include "storage/v3/isolation_level.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/storage.hpp" + +namespace memgraph::storage::v3::tests { +int64_t VerticesCount(Shard::Accessor &accessor) { + int64_t count{0}; + for ([[maybe_unused]] const auto &vertex : accessor.Vertices(View::NEW)) { + ++count; + } + + return count; +} + +inline constexpr std::array isolation_levels{IsolationLevel::SNAPSHOT_ISOLATION, IsolationLevel::READ_COMMITTED, + IsolationLevel::READ_UNCOMMITTED}; + +std::string_view IsolationLevelToString(const IsolationLevel isolation_level) { + switch (isolation_level) { + case IsolationLevel::SNAPSHOT_ISOLATION: + return "SNAPSHOT_ISOLATION"; + case IsolationLevel::READ_COMMITTED: + return "READ_COMMITTED"; + case IsolationLevel::READ_UNCOMMITTED: + return "READ_UNCOMMITTED"; + } +} + +class StorageIsolationLevelTest : public ::testing::TestWithParam { + protected: + [[nodiscard]] LabelId NameToLabelId(std::string_view label_name) { + return LabelId::FromUint(id_mapper.NameToId(label_name)); + } + + [[nodiscard]] PropertyId NameToPropertyId(std::string_view property_name) { + return PropertyId::FromUint(id_mapper.NameToId(property_name)); + } + + [[nodiscard]] EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) { + return EdgeTypeId::FromUint(id_mapper.NameToId(edge_type_name)); + } + + [[nodiscard]] coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += std::chrono::seconds(10); + return last_hlc; + } + + NameIdMapper id_mapper; + static constexpr int64_t min_primary_key_value{0}; + static constexpr int64_t max_primary_key_value{10000}; + const LabelId primary_label{NameToLabelId("label")}; + const PropertyId primary_property{NameToPropertyId("property")}; + coordinator::Hlc last_hlc{0, io::Time{}}; + + public: + struct PrintToStringParamName { + std::string operator()(const testing::TestParamInfo &info) { + return std::string(IsolationLevelToString(static_cast(info.param))); + } + }; +}; + +TEST_P(StorageIsolationLevelTest, Visibility) { + const auto default_isolation_level = GetParam(); + + for (auto override_isolation_level_index{0U}; override_isolation_level_index < isolation_levels.size(); + ++override_isolation_level_index) { + Shard store{primary_label, + {PropertyValue{min_primary_key_value}}, + std::vector{PropertyValue{max_primary_key_value}}, + Config{.transaction = {.isolation_level = default_isolation_level}}}; + ASSERT_TRUE( + store.CreateSchema(primary_label, {storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}})); + const auto override_isolation_level = isolation_levels[override_isolation_level_index]; + auto creator = store.Access(GetNextHlc()); + auto default_isolation_level_reader = store.Access(GetNextHlc()); + auto override_isolation_level_reader = store.Access(GetNextHlc(), override_isolation_level); + + ASSERT_EQ(VerticesCount(default_isolation_level_reader), 0); + ASSERT_EQ(VerticesCount(override_isolation_level_reader), 0); + + static constexpr auto iteration_count = 10; + { + SCOPED_TRACE(fmt::format( + "Visibility while the creator transaction is active " + "(default isolation level = {}, override isolation level = {})", + IsolationLevelToString(default_isolation_level), IsolationLevelToString(override_isolation_level))); + for (auto i{1}; i <= iteration_count; ++i) { + ASSERT_TRUE( + creator.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue{i}}}).HasValue()); + + const auto check_vertices_count = [i](auto &accessor, const auto isolation_level) { + const auto expected_count = isolation_level == IsolationLevel::READ_UNCOMMITTED ? i : 0; + EXPECT_EQ(VerticesCount(accessor), expected_count); + }; + check_vertices_count(default_isolation_level_reader, default_isolation_level); + check_vertices_count(override_isolation_level_reader, override_isolation_level); + } + } + + creator.Commit(GetNextHlc()); + { + SCOPED_TRACE(fmt::format( + "Visibility after the creator transaction is committed " + "(default isolation level = {}, override isolation level = {})", + IsolationLevelToString(default_isolation_level), IsolationLevelToString(override_isolation_level))); + const auto check_vertices_count = [](auto &accessor, const auto isolation_level) { + const auto expected_count = isolation_level == IsolationLevel::SNAPSHOT_ISOLATION ? 0 : iteration_count; + ASSERT_EQ(VerticesCount(accessor), expected_count); + }; + + check_vertices_count(default_isolation_level_reader, default_isolation_level); + check_vertices_count(override_isolation_level_reader, override_isolation_level); + } + + default_isolation_level_reader.Commit(GetNextHlc()); + override_isolation_level_reader.Commit(GetNextHlc()); + + SCOPED_TRACE("Visibility after a new transaction is started"); + auto verifier = store.Access(GetNextHlc()); + ASSERT_EQ(VerticesCount(verifier), iteration_count); + verifier.Commit(GetNextHlc()); + } +} + +INSTANTIATE_TEST_CASE_P(ParameterizedStorageIsolationLevelTests, StorageIsolationLevelTest, + ::testing::ValuesIn(isolation_levels), StorageIsolationLevelTest::PrintToStringParamName()); +} // namespace memgraph::storage::v3::tests diff --git a/tests/unit/storage_v3_vertex_accessors.cpp b/tests/unit/storage_v3_vertex_accessors.cpp index d9fb3cfd8..b5f6890d3 100644 --- a/tests/unit/storage_v3_vertex_accessors.cpp +++ b/tests/unit/storage_v3_vertex_accessors.cpp @@ -48,15 +48,22 @@ class StorageV3Accessor : public ::testing::Test { PropertyId NameToPropertyId(std::string_view property_name) { return storage.NameToProperty(property_name); } + coordinator::Hlc GetNextHlc() { + ++last_hlc.logical_id; + last_hlc.coordinator_wall_clock += std::chrono::seconds(10); + return last_hlc; + } + const std::vector pk{PropertyValue{0}}; const LabelId primary_label{LabelId::FromUint(1)}; const PropertyId primary_property{PropertyId::FromUint(2)}; Shard storage{primary_label, pk, std::nullopt}; + coordinator::Hlc last_hlc{0, io::Time{}}; }; TEST_F(StorageV3Accessor, TestPrimaryLabel) { { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(0)}}); ASSERT_TRUE(vertex.PrimaryLabel(View::NEW).HasValue()); const auto vertex_primary_label = vertex.PrimaryLabel(View::NEW).GetValue(); @@ -64,7 +71,7 @@ TEST_F(StorageV3Accessor, TestPrimaryLabel) { EXPECT_EQ(vertex_primary_label, primary_label); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}}); ASSERT_TRUE(vertex.PrimaryLabel(View::OLD).HasError()); const auto error_primary_label = vertex.PrimaryLabel(View::OLD).GetError(); @@ -72,12 +79,12 @@ TEST_F(StorageV3Accessor, TestPrimaryLabel) { EXPECT_EQ(error_primary_label, Error::NONEXISTENT_OBJECT); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}}); - ASSERT_FALSE(acc.Commit().HasError()); + acc.Commit(GetNextHlc()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto vertex = acc.FindVertex({PropertyValue{2}}, View::OLD); ASSERT_TRUE(vertex.has_value()); ASSERT_TRUE(acc.FindVertex({PropertyValue{2}}, View::NEW).has_value()); @@ -91,7 +98,7 @@ TEST_F(StorageV3Accessor, TestPrimaryLabel) { TEST_F(StorageV3Accessor, TestAddLabels) { storage.StoreMapping({{1, "label"}, {2, "property"}, {3, "label1"}, {4, "label2"}, {5, "label3"}}); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); const auto label2 = NameToLabelId("label2"); const auto label3 = NameToLabelId("label3"); @@ -102,7 +109,7 @@ TEST_F(StorageV3Accessor, TestAddLabels) { EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label2, label3)); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); const auto label2 = NameToLabelId("label2"); const auto label3 = NameToLabelId("label3"); @@ -117,7 +124,7 @@ TEST_F(StorageV3Accessor, TestAddLabels) { EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label2, label3)); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label"); auto vertex = acc.CreateVertexAndValidate(primary_label, {label1}, {{primary_property, PropertyValue(2)}}); ASSERT_TRUE(vertex.HasError()); @@ -126,7 +133,7 @@ TEST_F(StorageV3Accessor, TestAddLabels) { SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, label1)); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label"); auto vertex = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(3)}}); ASSERT_TRUE(vertex.HasValue()); @@ -142,7 +149,7 @@ TEST_F(StorageV3Accessor, TestRemoveLabels) { storage.StoreMapping({{1, "label"}, {2, "property"}, {3, "label1"}, {4, "label2"}, {5, "label3"}}); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); const auto label2 = NameToLabelId("label2"); const auto label3 = NameToLabelId("label3"); @@ -165,7 +172,7 @@ TEST_F(StorageV3Accessor, TestRemoveLabels) { EXPECT_TRUE(vertex.Labels(View::NEW).GetValue().empty()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const auto label1 = NameToLabelId("label1"); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}}); ASSERT_TRUE(vertex.Labels(View::NEW).HasValue()); @@ -175,7 +182,7 @@ TEST_F(StorageV3Accessor, TestRemoveLabels) { EXPECT_FALSE(res1.GetValue()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}}); const auto res1 = vertex.RemoveLabelAndValidate(primary_label); ASSERT_TRUE(res1.HasError()); @@ -189,14 +196,14 @@ TEST_F(StorageV3Accessor, TestSetKeysAndProperties) { storage.StoreMapping({{1, "label"}, {2, "property"}, {3, "prop1"}}); storage.StoreMapping({{1, "label"}, {2, "property"}, {3, "prop1"}}); { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); const PropertyId prop1{NameToPropertyId("prop1")}; auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(0)}}); const auto res = vertex.SetPropertyAndValidate(prop1, PropertyValue(1)); ASSERT_TRUE(res.HasValue()); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}}); const auto res = vertex.SetPropertyAndValidate(primary_property, PropertyValue(1)); ASSERT_TRUE(res.HasError()); @@ -206,7 +213,7 @@ TEST_F(StorageV3Accessor, TestSetKeysAndProperties) { SchemaProperty{primary_property, common::SchemaType::INT})); } { - auto acc = storage.Access(); + auto acc = storage.Access(GetNextHlc()); auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}}); const auto res = vertex.SetPropertyAndValidate(primary_property, PropertyValue()); ASSERT_TRUE(res.HasError());