tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'E118-MG-lexicographically-ordered-storage' into T1010-MG-implement-query-engine-client

Browse Source
This commit is contained in:
kostas kyrimis 2022-09-21 11:35:30 +00:00
commit f89af77be9
89 changed files with 16924 additions and 5828 deletions
src
expr
interpret
eval.hpp
typed_value.hpp
glue/v2
communication.cppcommunication.hpp
io
simulator
simulator_handle.hpp
transport.hpp
query/v2
accessors.cppaccessors.hpp
bindings
eval.hpp
common.hppcontext.hppconversions.hppdb_accessor.hpp
interpret
awesome_memgraph_functions.cpp
interpreter.cppinterpreter.hpp
plan
operator.cpp
procedure
mg_procedure_impl.cppmg_procedure_impl.hpppy_module.cpp
requests.hppshard_request_manager.hpp
stream
streams.cpp
trigger_context.cpptrigger_context.hpp
storage
v2
storage.cpp
v3
CMakeLists.txtcommit_log.cppcommit_log.hppconfig.hppconstraints.cppconstraints.hppconversions.hppdelta.hpp
durability
durability.cppdurability.hppsnapshot.cppsnapshot.hppwal.cppwal.hpp
edge.hppedge_accessor.cppedge_accessor.hppid_types.hppindices.cppindices.hppkey_store.cppkey_store.hpplexicographically_ordered_vertex.cpplexicographically_ordered_vertex.hppmvcc.hpp
replication
replication_client.cppreplication_client.hppreplication_server.cppreplication_server.hpp
schema_validator.cppschema_validator.hppschemas.cppschemas.hppshard.cppshard.hppshard_rsm.cppshard_rsm.hppstorage.cppstorage.hppvalue_conversions.hppvertex.hppvertex_accessor.cppvertex_accessor.hppvertex_id.hppvertices_skip_list.hpp
utils
concepts.hpp
tests
simulation
CMakeLists.txtcommon.hppshard_request_manager.cppshard_rsm.cpp
unit
CMakeLists.txtquery_v2_interpreter.cppquery_v2_query_plan_v2_create_set_remove_delete.cppresult_stream_faker.hppstorage_v3.cppstorage_v3_edge.cppstorage_v3_indices.cppstorage_v3_key_store.cppstorage_v3_property_store.cppstorage_v3_schema.cppstorage_v3_test_utils.cppstorage_v3_test_utils.hppstorage_v3_vertex_accessors.cpp

2
src/expr/interpret/eval.hpp
View File

@ -413,7 +413,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
template <typename VertexAccessor, typename TTag = Tag,
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
auto label = requests::Label{.id = LabelId::FromUint(label_ix.ix)};
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
auto has_label = vertex.HasLabel(label);
return !has_label;
}

2
src/expr/typed_value.hpp
View File

@ -117,10 +117,8 @@ class TypedValueT {
}
case TypedValueT::Type::Vertex:
return 34;
// return value.ValueVertex().Gid().AsUint();
case TypedValueT::Type::Edge:
return 35;
// return value.ValueEdge().Gid().AsUint();
case TypedValueT::Type::Path: {
// const auto &vertices = value.ValuePath().vertices();
// const auto &edges = value.ValuePath().edges();

24
src/glue/v2/communication.cpp
View File

@ -16,7 +16,7 @@
#include <vector>
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/storage.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/temporal.hpp"
@ -64,16 +64,16 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Storage &db,
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
@ -132,8 +132,9 @@ storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(vertex.Gid().AsUint());
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
@ -151,10 +152,11 @@ storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3:
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(edge.Gid().AsUint());
auto from = communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
auto to = communication::bolt::Id::FromUint(edge.ToVertex().Gid().AsUint());
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
@ -165,7 +167,7 @@ storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::Edg
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());

16
src/glue/v2/communication.hpp
View File

@ -28,36 +28,36 @@ namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param storage::v3::Storage for getting label and property names.
/// @param storage::v3::Shard for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view);
const storage::v3::Shard &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::v3::Storage for getting edge type and property names.
/// @param storage::v3::Shard for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view);
const storage::v3::Shard &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Storage &db, storage::v3::View view);
const storage::v3::Shard &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);

12
src/io/simulator/simulator_handle.hpp
View File

@ -55,15 +55,17 @@ class SimulatorHandle {
void TimeoutPromisesPastDeadline() {
const Time now = cluster_wide_time_microseconds_;
for (auto &[promise_key, dop] : promises_) {
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
if (dop.deadline < now) {
spdlog::debug("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
promise_key.replier_address.ToString());
spdlog::info("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
promise_key.replier_address.ToString());
std::move(dop).promise.TimeOut();
promises_.erase(promise_key);
it = promises_.erase(it);
stats_.timed_out_requests++;
} else {
++it;
}
}
}

2
src/io/transport.hpp
View File

@ -67,7 +67,7 @@ class Io {
I implementation_;
Address address_;
RequestId request_id_counter_ = 0;
Duration default_timeout_ = std::chrono::microseconds{50000};
Duration default_timeout_ = std::chrono::microseconds{100000};
public:
Io(I io, Address address) : implementation_(io), address_(address) {}

29
src/query/v2/accessors.cpp
View File

@ -13,12 +13,12 @@
#include "query/v2/requests.hpp"
namespace memgraph::query::v2::accessors {
EdgeAccessor::EdgeAccessor(Edge edge, std::map<std::string, Value> props)
EdgeAccessor::EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props)
: edge(std::move(edge)), properties(std::move(props)) {}
std::string EdgeAccessor::EdgeType() const { return edge.type.name; }
uint64_t EdgeAccessor::EdgeType() const { return edge.type.id; }
std::map<std::string, Value> EdgeAccessor::Properties() const {
std::vector<std::pair<PropertyId, Value>> EdgeAccessor::Properties() const {
return properties;
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
@ -27,18 +27,19 @@ std::map<std::string, Value> EdgeAccessor::Properties() const {
// return res;
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
MG_ASSERT(properties.contains(prop_name));
return properties[prop_name];
// TODO(kostasrim) fix this
return {};
}
requests::Edge EdgeAccessor::GetEdge() const { return edge; }
Edge EdgeAccessor::GetEdge() const { return edge; }
VertexAccessor EdgeAccessor::To() const { return VertexAccessor(Vertex{edge.dst}, {}); }
VertexAccessor EdgeAccessor::From() const { return VertexAccessor(Vertex{edge.src}, {}); }
VertexAccessor::VertexAccessor(Vertex v, std::map<requests::PropertyId, Value> props)
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props)
: vertex(std::move(v)), properties(std::move(props)) {}
std::vector<Label> VertexAccessor::Labels() const { return vertex.labels; }
@ -48,7 +49,7 @@ bool VertexAccessor::HasLabel(Label &label) const {
[label](const auto &l) { return l.id == label.id; }) != vertex.labels.end();
}
std::map<requests::PropertyId, Value> VertexAccessor::Properties() const {
std::vector<std::pair<PropertyId, Value>> VertexAccessor::Properties() const {
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
// res[name] = ValueToTypedValue(value);
@ -57,20 +58,18 @@ std::map<requests::PropertyId, Value> VertexAccessor::Properties() const {
return properties;
}
Value VertexAccessor::GetProperty(requests::PropertyId prop_id) const {
MG_ASSERT(properties.contains(prop_id));
return properties[prop_id];
Value VertexAccessor::GetProperty(PropertyId prop_id) const {
return std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; })->second;
// return ValueToTypedValue(properties[prop_name]);
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value VertexAccessor::GetProperty(const std::string & /*prop_name*/) const {
// TODO(kostasrim) Add string mapping
auto prop_id = requests::PropertyId::FromUint(0);
MG_ASSERT(properties.contains(prop_id));
return properties[prop_id];
return {};
// return ValueToTypedValue(properties[prop_name]);
}
requests::Vertex VertexAccessor::GetVertex() const { return vertex; }
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }
} // namespace memgraph::query::v2::accessors

32
src/query/v2/accessors.hpp
View File

@ -12,6 +12,8 @@
#pragma once
#include <optional>
#include <utility>
#include <vector>
#include "query/exceptions.hpp"
#include "query/v2/requests.hpp"
@ -23,24 +25,25 @@
namespace memgraph::query::v2::accessors {
using Value = requests::Value;
using Edge = requests::Edge;
using Vertex = requests::Vertex;
using Label = requests::Label;
using Value = memgraph::msgs::Value;
using Edge = memgraph::msgs::Edge;
using Vertex = memgraph::msgs::Vertex;
using Label = memgraph::msgs::Label;
using PropertyId = memgraph::msgs::PropertyId;
class VertexAccessor;
class EdgeAccessor final {
public:
EdgeAccessor(Edge edge, std::map<std::string, Value> props);
EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props);
std::string EdgeType() const;
uint64_t EdgeType() const;
std::map<std::string, Value> Properties() const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
Value GetProperty(const std::string &prop_name) const;
requests::Edge GetEdge() const;
Edge GetEdge() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
@ -60,24 +63,25 @@ class EdgeAccessor final {
private:
Edge edge;
mutable std::map<std::string, Value> properties;
std::vector<std::pair<PropertyId, Value>> properties;
};
class VertexAccessor final {
public:
using PropertyId = requests::PropertyId;
VertexAccessor(Vertex v, std::map<PropertyId, Value> props);
using PropertyId = msgs::PropertyId;
using Label = msgs::Label;
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props);
std::vector<Label> Labels() const;
bool HasLabel(Label &label) const;
std::map<PropertyId, Value> Properties() const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
Value GetProperty(PropertyId prop_id) const;
Value GetProperty(const std::string &prop_name) const;
requests::Vertex GetVertex() const;
msgs::Vertex GetVertex() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
@ -130,7 +134,7 @@ class VertexAccessor final {
private:
Vertex vertex;
mutable std::map<PropertyId, Value> properties;
std::vector<std::pair<PropertyId, Value>> properties;
};
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }

4
src/query/v2/bindings/eval.hpp
View File

@ -34,13 +34,13 @@ class Callable {
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
};
auto operator()(const requests::Value &val) const { return ValueToTypedValue(val); };
auto operator()(const msgs::Value &val) const { return ValueToTypedValue(val); };
};
} // namespace detail
using ExpressionEvaluator =
memgraph::expr::ExpressionEvaluator<TypedValue, EvaluationContext, DbAccessor, storage::v3::View,
storage::v3::LabelId, requests::Value, detail::Callable,
storage::v3::LabelId, msgs::Value, detail::Callable,
memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
} // namespace memgraph::query::v2

25
src/query/v2/common.hpp
View File

@ -88,6 +88,18 @@ concept AccessorWithSetProperty = requires(T accessor, const storage::v3::Proper
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
};
template <typename T>
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{
accessor.SetPropertyAndValidate(key, new_value)
} -> std::same_as<storage::v3::ResultSchema<storage::v3::PropertyValue>>;
};
template <typename TRecordAccessor>
concept RecordAccessor =
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
switch (schema_violation.status) {
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
@ -111,13 +123,14 @@ inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_vio
*schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_MODIFY_PRIMARY_LABEL: {
throw SchemaViolationException(fmt::format("Cannot add or remove label :{} since it is a primary label",
dba.LabelToName(schema_violation.label)));
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: {
throw SchemaViolationException(fmt::format(
"Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
throw SchemaViolationException(
fmt::format("Cannot create vertex with secondary label :{}", dba.LabelToName(schema_violation.label)));
throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}",
dba.LabelToName(schema_violation.label)));
}
}
}
@ -139,7 +152,7 @@ inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <AccessorWithSetProperty T>
template <RecordAccessor T>
storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
const TypedValue &value) {
try {

2
src/query/v2/context.hpp
View File

@ -73,7 +73,7 @@ struct ExecutionContext {
ExecutionStats execution_stats;
// TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
std::unique_ptr<requests::ShardRequestManagerInterface> shard_request_manager{nullptr};
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager{nullptr};
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");

28
src/query/v2/conversions.hpp
View File

@ -16,20 +16,20 @@
namespace memgraph::query::v2 {
inline TypedValue ValueToTypedValue(const requests::Value &value) {
using Value = requests::Value;
inline TypedValue ValueToTypedValue(const msgs::Value &value) {
using Value = msgs::Value;
switch (value.type) {
case Value::NILL:
case Value::Type::Null:
return {};
case Value::BOOL:
case Value::Type::Bool:
return TypedValue(value.bool_v);
case Value::INT64:
case Value::Type::Int64:
return TypedValue(value.int_v);
case Value::DOUBLE:
case Value::Type::Double:
return TypedValue(value.double_v);
case Value::STRING:
case Value::Type::String:
return TypedValue(value.string_v);
case Value::LIST: {
case Value::Type::List: {
const auto &lst = value.list_v;
std::vector<TypedValue> dst;
dst.reserve(lst.size());
@ -38,7 +38,7 @@ inline TypedValue ValueToTypedValue(const requests::Value &value) {
}
return TypedValue(std::move(dst));
}
case Value::MAP: {
case Value::Type::Map: {
const auto &value_map = value.map_v;
std::map<std::string, TypedValue> dst;
for (const auto &[key, val] : value_map) {
@ -46,18 +46,18 @@ inline TypedValue ValueToTypedValue(const requests::Value &value) {
}
return TypedValue(std::move(dst));
}
case Value::VERTEX:
case Value::Type::Vertex:
return TypedValue(accessors::VertexAccessor(value.vertex_v, {}));
case Value::EDGE:
case Value::Type::Edge:
return TypedValue(accessors::EdgeAccessor(value.edge_v, {}));
case Value::PATH:
case Value::Type::Path:
break;
}
throw std::runtime_error("Incorrect type in conversion");
}
inline requests::Value TypedValueToValue(const TypedValue &value) {
using Value = requests::Value;
inline msgs::Value TypedValueToValue(const TypedValue &value) {
using Value = msgs::Value;
switch (value.type()) {
case TypedValue::Type::Null:
return {};

72
src/query/v2/db_accessor.hpp
View File

@ -11,6 +11,7 @@
#pragma once
#include <cstdint>
#include <optional>
#include <vector>
@ -19,6 +20,7 @@
#include "query/v2/exceptions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
@ -109,13 +111,17 @@ class VertexAccessor final {
auto PrimaryLabel(storage::v3::View view) const { return impl_.PrimaryLabel(view); }
storage::v3::Result<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabel(label); }
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
storage::v3::ResultSchema<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabelAndValidate(label); }
storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::Result<bool> RemoveLabel(storage::v3::LabelId label) { return impl_.RemoveLabel(label); }
storage::v3::ResultSchema<bool> RemoveLabel(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
@ -132,9 +138,9 @@ class VertexAccessor final {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
storage::v3::ResultSchema<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
@ -162,7 +168,8 @@ class VertexAccessor final {
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
@ -179,7 +186,8 @@ class VertexAccessor final {
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
@ -188,9 +196,8 @@ class VertexAccessor final {
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
int64_t CypherId() const { return impl_.Gid().AsInt(); }
storage::v3::Gid Gid() const noexcept { return impl_.Gid(); }
// TODO(jbajic) Fix Remove Gid
static int64_t CypherId() { return 1; }
bool operator==(const VertexAccessor &v) const noexcept {
static_assert(noexcept(impl_ == v.impl_));
@ -200,14 +207,19 @@ class VertexAccessor final {
bool operator!=(const VertexAccessor &v) const noexcept { return !(*this == v); }
};
inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wnull-dereference"
// NOLINTNEXTLINE(readability-convert-member-functions-to-static,clang-analyzer-core.NonNullParamChecker)
inline VertexAccessor EdgeAccessor::To() const { return *static_cast<VertexAccessor *>(nullptr); }
inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static,clang-analyzer-core.NonNullParamChecker)
inline VertexAccessor EdgeAccessor::From() const { return *static_cast<VertexAccessor *>(nullptr); }
#pragma clang diagnostic pop
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
class DbAccessor final {
storage::v3::Storage::Accessor *accessor_;
storage::v3::Shard::Accessor *accessor_;
class VerticesIterable final {
storage::v3::VerticesIterable iterable_;
@ -239,10 +251,14 @@ class DbAccessor final {
};
public:
explicit DbAccessor(storage::v3::Storage::Accessor *accessor) : accessor_(accessor) {}
explicit DbAccessor(storage::v3::Shard::Accessor *accessor) : accessor_(accessor) {}
std::optional<VertexAccessor> FindVertex(storage::v3::Gid gid, storage::v3::View view) {
auto maybe_vertex = accessor_->FindVertex(gid, view);
// TODO(jbajic) Fix Remove Gid
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
std::optional<VertexAccessor> FindVertex(uint64_t /*unused*/) { return std::nullopt; }
std::optional<VertexAccessor> FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) {
auto maybe_vertex = accessor_->FindVertex(primary_key, view);
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
return std::nullopt;
}
@ -270,9 +286,6 @@ class DbAccessor final {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
// TODO Remove when query modules have been fixed
[[deprecated]] VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); }
storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
@ -285,13 +298,15 @@ class DbAccessor final {
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(),
to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(*maybe_edge);
}
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(&edge->impl_);
auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid());
if (res.HasError()) {
return res.GetError();
}
@ -342,11 +357,20 @@ class DbAccessor final {
return {std::make_optional<VertexAccessor>(*value)};
}
storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
// TODO(jbajic) Query engine should have a map of labels, properties and edge
// types
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::PropertyId NameToProperty(const std::string_view /*name*/) {
return storage::v3::PropertyId::FromUint(0);
}
storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::LabelId NameToLabel(const std::string_view /*name*/) { return storage::v3::LabelId::FromUint(0); }
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view /*name*/) {
return storage::v3::EdgeTypeId::FromUint(0);
}
const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }

2
src/query/v2/interpret/awesome_memgraph_functions.cpp
View File

@ -520,7 +520,7 @@ TypedValue Type(const TypedValue *args, int64_t nargs, const FunctionContext &ct
FType<Or<Null, Edge>>("type", args, nargs);
auto *dba = ctx.db_accessor;
if (args[0].IsNull()) return TypedValue(ctx.memory);
return TypedValue(args[0].ValueEdge().EdgeType(), ctx.memory);
return TypedValue(static_cast<int64_t>(args[0].ValueEdge().EdgeType()), ctx.memory);
}
TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {

37
src/query/v2/interpreter.cpp
View File

@ -42,6 +42,7 @@
#include "query/v2/plan/profile.hpp"
#include "query/v2/plan/vertex_count_cache.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/storage.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
@ -126,7 +127,7 @@ std::optional<std::string> GetOptionalStringValue(query::v2::Expression *express
class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
public:
explicit ReplQueryHandler(storage::v3::Storage *db) : db_(db) {}
explicit ReplQueryHandler(storage::v3::Shard *db) : db_(db) {}
/// @throw QueryRuntimeException if an error ocurred.
void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional<int64_t> port) override {
@ -254,7 +255,7 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
}
private:
storage::v3::Storage *db_;
storage::v3::Shard *db_;
};
/// returns false if the replication role can't be set
/// @throw QueryRuntimeException if an error ocurred.
@ -696,7 +697,7 @@ Callback HandleSchemaQuery(SchemaQuery *schema_query, InterpreterContext *interp
callback.header = {"property_name", "property_type"};
callback.fn = [interpreter_context, primary_label = schema_query->label_]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
const auto label = interpreter_context->NameToLabelId(primary_label.name);
const auto *schema = db->GetSchema(label);
std::vector<std::vector<TypedValue>> results;
if (schema) {
@ -721,11 +722,11 @@ Callback HandleSchemaQuery(SchemaQuery *schema_query, InterpreterContext *interp
callback.fn = [interpreter_context, primary_label = schema_query->label_,
schema_type_map = std::move(schema_type_map)]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
const auto label = interpreter_context->NameToLabelId(primary_label.name);
std::vector<storage::v3::SchemaProperty> schemas_types;
schemas_types.reserve(schema_type_map.size());
for (const auto &schema_type : schema_type_map) {
auto property_id = db->NameToProperty(schema_type.first.name);
auto property_id = interpreter_context->NameToPropertyId(schema_type.first.name);
schemas_types.push_back({property_id, schema_type.second});
}
if (!db->CreateSchema(label, schemas_types)) {
@ -740,7 +741,7 @@ Callback HandleSchemaQuery(SchemaQuery *schema_query, InterpreterContext *interp
case SchemaQuery::Action::DROP_SCHEMA: {
callback.fn = [interpreter_context, primary_label = schema_query->label_]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
const auto label = interpreter_context->NameToLabelId(primary_label.name);
if (!db->DropSchema(label)) {
throw QueryException(fmt::format("Schema on label :{} does not exist!", primary_label.name));
@ -922,7 +923,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
using RWType = plan::ReadWriteTypeChecker::RWType;
} // namespace
InterpreterContext::InterpreterContext(storage::v3::Storage *db, const InterpreterConfig config,
InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
const std::filesystem::path &data_directory)
// : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory /
// "streams"} {}
@ -943,8 +944,8 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
in_explicit_transaction_ = true;
expect_rollback_ = false;
db_accessor_ = std::make_unique<storage::v3::Storage::Accessor>(
interpreter_context_->db->Access(GetIsolationLevelOverride()));
db_accessor_ =
std::make_unique<storage::v3::Shard::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
execution_db_accessor_.emplace(db_accessor_.get());
// if (interpreter_context_->trigger_store.HasTriggers()) {
@ -1216,14 +1217,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
}
};
auto label = interpreter_context->db->NameToLabel(index_query->label_.name);
auto label = interpreter_context->NameToLabelId(index_query->label_.name);
std::vector<storage::v3::PropertyId> properties;
std::vector<std::string> properties_string;
properties.reserve(index_query->properties_.size());
properties_string.reserve(index_query->properties_.size());
for (const auto &prop : index_query->properties_) {
properties.push_back(interpreter_context->db->NameToProperty(prop.name));
properties.push_back(interpreter_context->NameToPropertyId(prop.name));
properties_string.push_back(prop.name);
}
auto properties_stringified = utils::Join(properties_string, ", ");
@ -1467,7 +1468,7 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
[interpreter_context](AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
if (auto maybe_error = interpreter_context->db->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::v3::Storage::CreateSnapshotError::DisabledForReplica:
case storage::v3::Shard::CreateSnapshotError::DisabledForReplica:
throw utils::BasicException(
"Failed to create a snapshot. Replica instances are not allowed to create them.");
}
@ -1522,8 +1523,8 @@ PreparedQuery PrepareVersionQuery(ParsedQuery parsed_query, const bool in_explic
}
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::v3::Storage *db, utils::MemoryResource *execution_memory) {
std::map<std::string, TypedValue> * /*summary*/, InterpreterContext *interpreter_context,
storage::v3::Shard *db, utils::MemoryResource * /*execution_memory*/) {
if (in_explicit_transaction) {
throw InfoInMulticommandTxException();
}
@ -1619,13 +1620,13 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_
auto *constraint_query = utils::Downcast<ConstraintQuery>(parsed_query.query);
std::function<void(Notification &)> handler;
auto label = interpreter_context->db->NameToLabel(constraint_query->constraint_.label.name);
auto label = interpreter_context->NameToLabelId(constraint_query->constraint_.label.name);
std::vector<storage::v3::PropertyId> properties;
std::vector<std::string> properties_string;
properties.reserve(constraint_query->constraint_.properties.size());
properties_string.reserve(constraint_query->constraint_.properties.size());
for (const auto &prop : constraint_query->constraint_.properties) {
properties.push_back(interpreter_context->db->NameToProperty(prop.name));
properties.push_back(interpreter_context->NameToPropertyId(prop.name));
properties_string.push_back(prop.name);
}
auto properties_stringified = utils::Join(properties_string, ", ");
@ -1884,8 +1885,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
(utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
utils::Downcast<TriggerQuery>(parsed_query.query))) {
db_accessor_ = std::make_unique<storage::v3::Storage::Accessor>(
interpreter_context_->db->Access(GetIsolationLevelOverride()));
db_accessor_ =
std::make_unique<storage::v3::Shard::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
execution_db_accessor_.emplace(db_accessor_.get());
}

25
src/query/v2/interpreter.hpp
View File

@ -27,9 +27,8 @@
#include "query/v2/plan/operator.hpp"
#include "query/v2/plan/read_write_type_checker.hpp"
#include "query/v2/stream.hpp"
//#include "query/v2/stream/streams.hpp"
//#include "query/v2/trigger.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
@ -165,10 +164,10 @@ struct PreparedQuery {
* been passed to an `Interpreter` instance.
*/
struct InterpreterContext {
explicit InterpreterContext(storage::v3::Storage *db, InterpreterConfig config,
explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
const std::filesystem::path &data_directory);
storage::v3::Storage *db;
storage::v3::Shard *db;
std::optional<double> tsc_frequency{utils::GetTSCFrequency()};
std::atomic<bool> is_shutting_down{false};
@ -180,6 +179,22 @@ struct InterpreterContext {
utils::SkipList<PlanCacheEntry> plan_cache;
const InterpreterConfig config;
storage::v3::LabelId NameToLabelId(std::string_view label_name) {
return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
}
storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
}
storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
}
private:
// TODO Replace with local map of labels, properties and edge type ids
storage::v3::NameIdMapper query_id_mapper;
};
/// Function that is used to tell all active interpreters that they should stop
@ -310,7 +325,7 @@ class Interpreter final {
// This cannot be std::optional because we need to move this accessor later on into a lambda capture
// which is assigned to std::function. std::function requires every object to be copyable, so we
// move this unique_ptr into a shrared_ptr.
std::unique_ptr<storage::v3::Storage::Accessor> db_accessor_;
std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
std::optional<DbAccessor> execution_db_accessor_;
bool in_explicit_transaction_{false};
bool expect_rollback_{false};

145
src/query/v2/plan/operator.cpp
View File

@ -26,6 +26,7 @@
#include <cppitertools/chain.hpp>
#include <cppitertools/imap.hpp>
#include "expr/exceptions.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/bindings/eval.hpp"
#include "query/v2/bindings/symbol_table.hpp"
@ -261,8 +262,8 @@ class ScanAllCursor : public Cursor {
std::optional<typename std::result_of<TVerticesFun(Frame &, ExecutionContext &)>::type::value_type> vertices_;
std::optional<decltype(vertices_.value().begin())> vertices_it_;
const char *op_name_;
std::vector<requests::ScanVerticesResponse> current_batch;
requests::ExecutionState<requests::ScanVerticesRequest> request_state;
std::vector<msgs::ScanVerticesResponse> current_batch;
msgs::ExecutionState<msgs::ScanVerticesRequest> request_state;
};
ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, storage::v3::View view)
@ -429,17 +430,16 @@ ACCEPT_WITH_INPUT(ScanAllById)
UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByIdOperator);
// TODO Reimplement when we have reliable conversion between hash value and pk
auto vertices = [this](Frame &frame, ExecutionContext &context) -> std::optional<std::vector<VertexAccessor>> {
auto *db = context.db_accessor;
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_);
auto value = expression_->Accept(evaluator);
if (!value.IsNumeric()) return std::nullopt;
int64_t id = value.IsInt() ? value.ValueInt() : value.ValueDouble();
if (value.IsDouble() && id != value.ValueDouble()) return std::nullopt;
auto maybe_vertex = db->FindVertex(storage::v3::Gid::FromInt(id), view_);
if (!maybe_vertex) return std::nullopt;
return std::vector<VertexAccessor>{*maybe_vertex};
// auto *db = context.db_accessor;
// ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
// view_); auto value = expression_->Accept(evaluator); if (!value.IsNumeric()) return std::nullopt; int64_t id =
// value.IsInt() ? value.ValueInt() : value.ValueDouble(); if (value.IsDouble() && id != value.ValueDouble()) return
// std::nullopt; auto maybe_vertex = db->FindVertex(storage::v3::Gid::FromInt(id), view_); auto maybe_vertex =
// nullptr; if (!maybe_vertex) return std::nullopt;
return std::nullopt;
// return std::vector<VertexAccessor>{*maybe_vertex};
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllById");
@ -982,102 +982,6 @@ concept AccessorWithProperties = requires(T value, storage::v3::PropertyId prope
{value.SetProperty(property_id, property_value)};
};
/// Helper function that sets the given values on either a Vertex or an Edge.
///
/// @tparam TRecordAccessor Either RecordAccessor<Vertex> or
/// RecordAccessor<Edge>
template <AccessorWithProperties TRecordAccessor>
void SetPropertiesOnRecord(TRecordAccessor *record, const TypedValue &rhs, SetProperties::Op op,
ExecutionContext *context) {
std::optional<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> old_values;
if (op == SetProperties::Op::REPLACE) {
auto maybe_value = record->ClearProperties();
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted graph element.");
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException("Unexpected error when setting properties.");
}
}
}
auto get_props = [](const auto &record) {
auto maybe_props = record.Properties(storage::v3::View::NEW);
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from an object that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Unexpected error when getting properties.");
}
}
return *maybe_props;
};
auto register_set_property = [&](auto &&returned_old_value, auto key, auto &&new_value) {
auto old_value = storage::v3::PropertyToTypedValue<TypedValue>([&]() -> storage::v3::PropertyValue {
if (!old_values) {
return std::forward<decltype(returned_old_value)>(returned_old_value);
}
if (auto it = old_values->find(key); it != old_values->end()) {
return std::move(it->second);
}
return {};
}());
};
auto set_props = [&, record](auto properties) {
for (auto &kv : properties) {
auto maybe_error = record->SetProperty(kv.first, kv.second);
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted graph element.");
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException("Unexpected error when setting properties.");
}
}
}
};
switch (rhs.type()) {
case TypedValue::Type::Edge:
set_props(get_props(rhs.ValueEdge()));
break;
case TypedValue::Type::Vertex:
set_props(get_props(rhs.ValueVertex()));
break;
case TypedValue::Type::Map: {
for (const auto &kv : rhs.ValueMap()) {
auto key = context->db_accessor->NameToProperty(kv.first);
auto old_value = PropsSetChecked(record, *context->db_accessor, key, kv.second);
}
break;
}
default:
throw QueryRuntimeException(
"Right-hand side in SET expression must be a node, an edge or a "
"map.");
}
}
} // namespace
bool SetProperties::SetPropertiesCursor::Pull(Frame &frame, ExecutionContext &context) {
@ -2805,7 +2709,7 @@ class DistributedScanAllCursor : public Cursor {
using VertexAccessor = accessors::VertexAccessor;
bool MakeRequest(requests::ShardRequestManagerInterface &shard_manager) {
bool MakeRequest(msgs::ShardRequestManagerInterface &shard_manager) {
current_batch = shard_manager.Request(request_state_);
current_vertex_it = current_batch.begin();
return !current_batch.empty();
@ -2815,7 +2719,7 @@ class DistributedScanAllCursor : public Cursor {
SCOPED_PROFILE_OP(op_name_);
auto &shard_manager = *context.shard_request_manager;
if (MustAbort(context)) throw HintedAbortError();
using State = requests::ExecutionState<requests::ScanVerticesRequest>;
using State = msgs::ExecutionState<msgs::ScanVerticesRequest>;
if (request_state_.state == State::INITIALIZING) {
if (!input_cursor_->Pull(frame, context)) return false;
@ -2838,7 +2742,7 @@ class DistributedScanAllCursor : public Cursor {
void ResetExecutionState() {
current_batch.clear();
current_vertex_it = current_batch.end();
request_state_ = requests::ExecutionState<requests::ScanVerticesRequest>{};
request_state_ = msgs::ExecutionState<msgs::ScanVerticesRequest>{};
}
void Reset() override {
@ -2852,7 +2756,7 @@ class DistributedScanAllCursor : public Cursor {
const char *op_name_;
std::vector<VertexAccessor> current_batch;
decltype(std::vector<VertexAccessor>().begin()) current_vertex_it;
requests::ExecutionState<requests::ScanVerticesRequest> request_state_;
msgs::ExecutionState<msgs::ScanVerticesRequest> request_state_;
};
class DistributedCreateNodeCursor : public Cursor {
@ -2877,11 +2781,11 @@ class DistributedCreateNodeCursor : public Cursor {
void Reset() override { state_ = {}; }
std::vector<requests::NewVertex> NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) const {
std::vector<requests::NewVertex> requests;
std::vector<msgs::NewVertex> NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) const {
std::vector<msgs::NewVertex> requests;
for (const auto &node_info : nodes_info_) {
requests::NewVertex rqst;
std::map<requests::PropertyId, requests::Value> properties;
msgs::NewVertex rqst;
std::map<msgs::PropertyId, msgs::Value> properties;
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr,
storage::v3::View::NEW);
if (const auto *node_info_properties = std::get_if<PropertiesMapList>(&node_info.properties)) {
@ -2889,7 +2793,7 @@ class DistributedCreateNodeCursor : public Cursor {
TypedValue val = value_expression->Accept(evaluator);
properties[key] = TypedValueToValue(val);
if (context.shard_request_manager->IsPrimaryKey(key)) {
rqst.primary_key.push_back(storage::v3::TypedToPropertyValue(val));
rqst.primary_key.push_back(storage::v3::TypedValueToValue(val));
}
}
} else {
@ -2899,7 +2803,7 @@ class DistributedCreateNodeCursor : public Cursor {
auto property_id = context.shard_request_manager->NameToProperty(key_str);
properties[property_id] = TypedValueToValue(value);
if (context.shard_request_manager->IsPrimaryKey(property_id)) {
rqst.primary_key.push_back(storage::v3::TypedToPropertyValue(value));
rqst.primary_key.push_back(storage::v3::TypedValueToValue(value));
}
}
}
@ -2907,7 +2811,8 @@ class DistributedCreateNodeCursor : public Cursor {
if (node_info.labels.empty()) {
throw QueryRuntimeException("Primary label must be defined!");
}
rqst.label_ids = requests::Label{node_info.labels[0]};
// TODO(kostasrim) Copy non primary labels as well
rqst.label_ids.push_back(msgs::Label{node_info.labels[0]});
requests.push_back(std::move(rqst));
}
return requests;
@ -2916,6 +2821,6 @@ class DistributedCreateNodeCursor : public Cursor {
private:
const UniqueCursorPtr input_cursor_;
std::vector<NodeCreationInfo> nodes_info_;
requests::ExecutionState<requests::CreateVerticesRequest> state_;
msgs::ExecutionState<msgs::CreateVerticesRequest> state_;
};
} // namespace memgraph::query::v2::plan

128
src/query/v2/procedure/mg_procedure_impl.cpp
View File

@ -29,6 +29,7 @@
#include "query/v2/procedure/mg_procedure_helpers.hpp"
#include "query/v2/stream/common.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/view.hpp"
#include "utils/algorithm.hpp"
@ -1510,8 +1511,9 @@ mgp_error mgp_properties_iterator_next(mgp_properties_iterator *it, mgp_property
result);
}
// TODO(jbajic) Fix Remove Gid
mgp_error mgp_vertex_get_id(mgp_vertex *v, mgp_vertex_id *result) {
return WrapExceptions([v] { return mgp_vertex_id{.as_int = v->impl.Gid().AsInt()}; }, result);
return WrapExceptions([] { return mgp_vertex_id{.as_int = 0}; }, result);
}
mgp_error mgp_vertex_underlying_graph_is_mutable(mgp_vertex *v, int *result) {
@ -1586,17 +1588,19 @@ mgp_error mgp_vertex_set_property(struct mgp_vertex *v, const char *property_nam
const auto prop_key = v->graph->impl->NameToProperty(property_name);
const auto result = v->impl.SetProperty(prop_key, ToPropertyValue(*property_value));
if (result.HasError()) {
switch (result.GetError()) {
case memgraph::storage::v3::Error::DELETED_OBJECT:
throw DeletedObjectException{"Cannot set the properties of a deleted vertex!"};
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
LOG_FATAL("Query modules shouldn't have access to nonexistent objects when setting a property of a vertex!");
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
LOG_FATAL("Unexpected error when setting a property of a vertex.");
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
throw SerializationException{"Cannot serialize setting a property of a vertex."};
}
// TODO(jbajic) Fix query modules
// switch (result.GetError()) {
// case memgraph::storage::v3::Error::DELETED_OBJECT:
// throw DeletedObjectException{"Cannot set the properties of a deleted vertex!"};
// case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
// LOG_FATAL("Query modules shouldn't have access to nonexistent objects when setting a property of a
// vertex!");
// case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
// case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
// LOG_FATAL("Unexpected error when setting a property of a vertex.");
// case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
// throw SerializationException{"Cannot serialize setting a property of a vertex."};
// }
}
auto &ctx = v->graph->ctx;
@ -1628,17 +1632,18 @@ mgp_error mgp_vertex_add_label(struct mgp_vertex *v, mgp_label label) {
const auto result = v->impl.AddLabel(label_id);
if (result.HasError()) {
switch (result.GetError()) {
case memgraph::storage::v3::Error::DELETED_OBJECT:
throw DeletedObjectException{"Cannot add a label to a deleted vertex!"};
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
LOG_FATAL("Query modules shouldn't have access to nonexistent objects when adding a label to a vertex!");
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
LOG_FATAL("Unexpected error when adding a label to a vertex.");
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
throw SerializationException{"Cannot serialize adding a label to a vertex."};
}
// TODO(jbajic) Fix query modules
// switch (result.GetError()) {
// case memgraph::storage::v3::Error::DELETED_OBJECT:
// throw DeletedObjectException{"Cannot add a label to a deleted vertex!"};
// case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
// LOG_FATAL("Query modules shouldn't have access to nonexistent objects when adding a label to a vertex!");
// case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
// case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
// LOG_FATAL("Unexpected error when adding a label to a vertex.");
// case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
// throw SerializationException{"Cannot serialize adding a label to a vertex."};
// }
}
auto &ctx = v->graph->ctx;
@ -1660,17 +1665,19 @@ mgp_error mgp_vertex_remove_label(struct mgp_vertex *v, mgp_label label) {
const auto result = v->impl.RemoveLabel(label_id);
if (result.HasError()) {
switch (result.GetError()) {
case memgraph::storage::v3::Error::DELETED_OBJECT:
throw DeletedObjectException{"Cannot remove a label from a deleted vertex!"};
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing a label from a vertex!");
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
LOG_FATAL("Unexpected error when removing a label from a vertex.");
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
throw SerializationException{"Cannot serialize removing a label from a vertex."};
}
// TODO(jbajic) Fix query modules
// switch (result.GetError()) {
// case memgraph::storage::v3::Error::DELETED_OBJECT:
// throw DeletedObjectException{"Cannot remove a label from a deleted vertex!"};
// case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
// LOG_FATAL("Query modules shouldn't have access to nonexistent objects when removing a label from a
// vertex!");
// case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
// case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
// LOG_FATAL("Unexpected error when removing a label from a vertex.");
// case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
// throw SerializationException{"Cannot serialize removing a label from a vertex."};
// }
}
auto &ctx = v->graph->ctx;
@ -2074,11 +2081,12 @@ mgp_error mgp_edge_iter_properties(mgp_edge *e, mgp_memory *memory, mgp_properti
mgp_error mgp_graph_get_vertex_by_id(mgp_graph *graph, mgp_vertex_id id, mgp_memory *memory, mgp_vertex **result) {
return WrapExceptions(
[graph, id, memory]() -> mgp_vertex * {
auto maybe_vertex = graph->impl->FindVertex(memgraph::storage::v3::Gid::FromInt(id.as_int), graph->view);
if (maybe_vertex) {
return NewRawMgpObject<mgp_vertex>(memory, *maybe_vertex, graph);
}
[]() -> mgp_vertex * {
// TODO(jbajic) Fix Remove Gid
// auto maybe_vertex = graph->impl->FindVertex(0);
// if (maybe_vertex) {
// return NewRawMgpObject<mgp_vertex>(memory, *maybe_vertex, graph);
// }
return nullptr;
},
result);
@ -2089,23 +2097,25 @@ mgp_error mgp_graph_is_mutable(mgp_graph *graph, int *result) {
return mgp_error::MGP_ERROR_NO_ERROR;
};
mgp_error mgp_graph_create_vertex(struct mgp_graph *graph, mgp_memory *memory, mgp_vertex **result) {
return WrapExceptions(
[=] {
if (!MgpGraphIsMutable(*graph)) {
throw ImmutableObjectException{"Cannot create a vertex in an immutable graph!"};
}
auto vertex = graph->impl->InsertVertex();
// TODO(jbajic) Fix Remove Gid
mgp_error mgp_graph_create_vertex(struct mgp_graph * /*graph*/, mgp_memory * /*memory*/, mgp_vertex ** /*result*/) {
// return WrapExceptions(
// [=] {
// if (!MgpGraphIsMutable(*graph)) {
// throw ImmutableObjectException{"Cannot create a vertex in an immutable graph!"};
// }
// auto vertex = graph->impl->InsertVertex();
auto &ctx = graph->ctx;
ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::CREATED_NODES] += 1;
// auto &ctx = graph->ctx;
// ctx->execution_stats[memgraph::query::v2::ExecutionStats::Key::CREATED_NODES] += 1;
if (ctx->trigger_context_collector) {
ctx->trigger_context_collector->RegisterCreatedObject(vertex);
}
return NewRawMgpObject<mgp_vertex>(memory, vertex, graph);
},
result);
// if (ctx->trigger_context_collector) {
// ctx->trigger_context_collector->RegisterCreatedObject(vertex);
// }
// return NewRawMgpObject<mgp_vertex>(memory, nullptr, graph);
// },
// result);
return mgp_error::MGP_ERROR_NO_ERROR;
}
mgp_error mgp_graph_delete_vertex(struct mgp_graph *graph, mgp_vertex *vertex) {
@ -2177,11 +2187,11 @@ mgp_error mgp_graph_detach_delete_vertex(struct mgp_graph *graph, mgp_vertex *ve
if (!trigger_ctx_collector) {
return;
}
trigger_ctx_collector->RegisterDeletedObject((*result)->first);
if (!trigger_ctx_collector->ShouldRegisterDeletedObject<memgraph::query::v2::EdgeAccessor>()) {
return;
}
// TODO(jbajic) Fix Remove Gid
// trigger_ctx_collector->RegisterDeletedObject((*result)->first);
// if (!trigger_ctx_collector->ShouldRegisterDeletedObject<memgraph::query::v2::EdgeAccessor>()) {
// return;
// }
for (const auto &edge : (*result)->second) {
trigger_ctx_collector->RegisterDeletedObject(edge);
}

3
src/query/v2/procedure/mg_procedure_impl.hpp
View File

@ -474,9 +474,10 @@ struct mgp_edge {
/// the allocator which was used to allocate `this`.
using allocator_type = memgraph::utils::Allocator<mgp_edge>;
// TODO(antaljanosbenjamin): Handle this static assert failure when we will support procedures again
// Hopefully EdgeAccessor copy constructor remains noexcept, so that we can
// have everything noexcept here.
static_assert(std::is_nothrow_copy_constructible_v<memgraph::query::v2::EdgeAccessor>);
// static_assert(std::is_nothrow_copy_constructible_v<memgraph::query::v2::EdgeAccessor>);
static mgp_edge *Copy(const mgp_edge &edge, mgp_memory &memory);

7
src/query/v2/procedure/py_module.cpp
View File

@ -336,9 +336,10 @@ PyObject *PyGraphCreateVertex(PyGraph *self, PyObject *Py_UNUSED(ignored)) {
MG_ASSERT(PyGraphIsValidImpl(*self));
MG_ASSERT(self->memory);
MgpUniquePtr<mgp_vertex> new_vertex{nullptr, mgp_vertex_destroy};
if (RaiseExceptionFromErrorCode(CreateMgpObject(new_vertex, mgp_graph_create_vertex, self->graph, self->memory))) {
return nullptr;
}
// TODO(jbajic) Fix query module
// if (RaiseExceptionFromErrorCode(CreateMgpObject(new_vertex, mgp_graph_create_vertex, self->graph, self->memory))) {
// return nullptr;
// }
auto *py_vertex = MakePyVertexWithoutCopy(*new_vertex, self);
if (py_vertex != nullptr) {
static_cast<void>(new_vertex.release());

513
src/query/v2/requests.hpp
View File

@ -17,6 +17,7 @@
#include <memory>
#include <optional>
#include <unordered_map>
#include <utility>
#include <variant>
#include <vector>
@ -24,35 +25,35 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
namespace requests {
namespace memgraph::msgs {
using coordinator::Hlc;
using storage::v3::LabelId;
struct Value;
struct Label {
using LabelId = memgraph::storage::v3::LabelId;
LabelId id;
friend bool operator==(const Label &lhs, const Label &rhs) { return lhs.id == rhs.id; }
};
// TODO(kostasrim) update this with CompoundKey, same for the rest of the file.
using PrimaryKey = std::vector<memgraph::storage::v3::PropertyValue>;
using PrimaryKey = std::vector<Value>;
using VertexId = std::pair<Label, PrimaryKey>;
struct VertexId {
Label primary_label;
PrimaryKey primary_key;
friend bool operator==(const VertexId &lhs, const VertexId &rhs) {
return (lhs.primary_label == rhs.primary_label) && (lhs.primary_key == rhs.primary_key);
}
};
inline bool operator==(const VertexId &lhs, const VertexId &rhs) {
return (lhs.first == rhs.first) && (lhs.second == rhs.second);
}
using Gid = size_t;
using PropertyId = memgraph::storage::v3::PropertyId;
struct EdgeType {
std::string name;
uint64_t id;
friend bool operator==(const EdgeType &lhs, const EdgeType &rhs) = default;
};
struct EdgeId {
VertexId id;
Gid gid;
};
@ -67,6 +68,7 @@ struct Vertex {
struct Edge {
VertexId src;
VertexId dst;
EdgeId id;
EdgeType type;
friend bool operator==(const Edge &lhs, const Edge &rhs) {
return (lhs.src == rhs.src) && (lhs.dst == rhs.dst) && (lhs.type == rhs.type);
@ -86,7 +88,221 @@ struct Path {
struct Null {};
struct Value {
enum Type { NILL, BOOL, INT64, DOUBLE, STRING, LIST, MAP, VERTEX, EDGE, PATH };
Value() : null_v{} {}
explicit Value(const bool val) : type(Type::Bool), bool_v(val) {}
explicit Value(const int64_t val) : type(Type::Int64), int_v(val) {}
explicit Value(const double val) : type(Type::Double), double_v(val) {}
explicit Value(const Vertex val) : type(Type::Vertex), vertex_v(val) {}
explicit Value(const Edge val) : type(Type::Edge), edge_v(val) {}
explicit Value(const std::string &val) : type(Type::String) { new (&string_v) std::string(val); }
explicit Value(const char *val) : type(Type::String) { new (&string_v) std::string(val); }
explicit Value(const std::vector<Value> &val) : type(Type::List) { new (&list_v) std::vector<Value>(val); }
explicit Value(const std::map<std::string, Value> &val) : type(Type::Map) {
new (&map_v) std::map<std::string, Value>(val);
}
explicit Value(std::string &&val) noexcept : type(Type::String) { new (&string_v) std::string(std::move(val)); }
explicit Value(std::vector<Value> &&val) noexcept : type(Type::List) {
new (&list_v) std::vector<Value>(std::move(val));
}
explicit Value(std::map<std::string, Value> &&val) noexcept : type(Type::Map) {
new (&map_v) std::map<std::string, Value>(std::move(val));
}
~Value() { DestroyValue(); }
void DestroyValue() noexcept {
switch (type) {
case Type::Null:
case Type::Bool:
case Type::Int64:
case Type::Double:
return;
case Type::String:
std::destroy_at(&string_v);
return;
case Type::List:
std::destroy_at(&list_v);
return;
case Type::Map:
std::destroy_at(&map_v);
return;
case Type::Vertex:
std::destroy_at(&vertex_v);
return;
case Type::Path:
std::destroy_at(&path_v);
return;
case Type::Edge:
std::destroy_at(&edge_v);
}
}
Value(const Value &other) : type(other.type) {
switch (other.type) {
case Type::Null:
return;
case Type::Bool:
this->bool_v = other.bool_v;
return;
case Type::Int64:
this->int_v = other.int_v;
return;
case Type::Double:
this->double_v = other.double_v;
return;
case Type::String:
new (&string_v) std::string(other.string_v);
return;
case Type::List:
new (&list_v) std::vector<Value>(other.list_v);
return;
case Type::Map:
new (&map_v) std::map<std::string, Value>(other.map_v);
return;
case Type::Vertex:
new (&vertex_v) Vertex(other.vertex_v);
return;
case Type::Edge:
new (&edge_v) Edge(other.edge_v);
return;
case Type::Path:
new (&path_v) Path(other.path_v);
return;
}
}
Value(Value &&other) noexcept : type(other.type) {
switch (other.type) {
case Type::Null:
break;
case Type::Bool:
this->bool_v = other.bool_v;
break;
case Type::Int64:
this->int_v = other.int_v;
break;
case Type::Double:
this->double_v = other.double_v;
break;
case Type::String:
new (&string_v) std::string(std::move(other.string_v));
break;
case Type::List:
new (&list_v) std::vector<Value>(std::move(other.list_v));
break;
case Type::Map:
new (&map_v) std::map<std::string, Value>(std::move(other.map_v));
break;
case Type::Vertex:
new (&vertex_v) Vertex(std::move(other.vertex_v));
break;
case Type::Edge:
new (&edge_v) Edge(std::move(other.edge_v));
break;
case Type::Path:
new (&path_v) Path(std::move(other.path_v));
break;
}
other.DestroyValue();
other.type = Type::Null;
}
Value &operator=(const Value &other) {
if (this == &other) return *this;
DestroyValue();
type = other.type;
switch (other.type) {
case Type::Null:
break;
case Type::Bool:
this->bool_v = other.bool_v;
break;
case Type::Int64:
this->int_v = other.int_v;
break;
case Type::Double:
this->double_v = other.double_v;
break;
case Type::String:
new (&string_v) std::string(other.string_v);
break;
case Type::List:
new (&list_v) std::vector<Value>(other.list_v);
break;
case Type::Map:
new (&map_v) std::map<std::string, Value>(other.map_v);
break;
case Type::Vertex:
new (&vertex_v) Vertex(other.vertex_v);
break;
case Type::Edge:
new (&edge_v) Edge(other.edge_v);
break;
case Type::Path:
new (&path_v) Path(other.path_v);
break;
}
return *this;
}
Value &operator=(Value &&other) noexcept {
if (this == &other) return *this;
DestroyValue();
type = other.type;
switch (other.type) {
case Type::Null:
break;
case Type::Bool:
this->bool_v = other.bool_v;
break;
case Type::Int64:
this->int_v = other.int_v;
break;
case Type::Double:
this->double_v = other.double_v;
break;
case Type::String:
new (&string_v) std::string(std::move(other.string_v));
break;
case Type::List:
new (&list_v) std::vector<Value>(std::move(other.list_v));
break;
case Type::Map:
new (&map_v) std::map<std::string, Value>(std::move(other.map_v));
break;
case Type::Vertex:
new (&vertex_v) Vertex(std::move(other.vertex_v));
break;
case Type::Edge:
new (&edge_v) Edge(std::move(other.edge_v));
break;
case Type::Path:
new (&path_v) Path(std::move(other.path_v));
break;
}
other.DestroyValue();
other.type = Type::Null;
return *this;
}
enum class Type : uint8_t { Null, Bool, Int64, Double, String, List, Map, Vertex, Edge, Path };
Type type{Type::Null};
union {
Null null_v;
bool bool_v;
@ -100,194 +316,33 @@ struct Value {
Path path_v;
};
Type type;
Value() : null_v{}, type(NILL) {}
// copy ctor needed.
Value(const Value &value) : type(value.type) {
switch (value.type) {
case NILL:
null_v = {};
break;
case BOOL:
bool_v = value.bool_v;
break;
case INT64:
int_v = value.int_v;
break;
case DOUBLE:
double_v = value.double_v;
break;
case STRING:
string_v = value.string_v;
break;
case LIST:
list_v = value.list_v;
break;
case MAP:
map_v = value.map_v;
break;
case VERTEX:
vertex_v = value.vertex_v;
break;
case EDGE:
edge_v = value.edge_v;
break;
case PATH:
path_v = value.path_v;
break;
}
}
Value(Value &&other) noexcept : type(other.type) {
switch (other.type) {
case Type::NILL:
break;
case Type::BOOL:
this->bool_v = other.bool_v;
break;
case Type::INT64:
this->int_v = other.int_v;
break;
case Type::DOUBLE:
this->double_v = other.double_v;
break;
case Type::STRING:
new (&string_v) std::string(std::move(other.string_v));
break;
case Type::LIST:
new (&list_v) std::vector<Value>(std::move(other.list_v));
break;
case Type::MAP:
new (&map_v) std::map<std::string, Value>(std::move(other.map_v));
break;
case Type::VERTEX:
new (&vertex_v) Vertex(std::move(other.vertex_v));
break;
case Type::EDGE:
new (&edge_v) Edge(std::move(other.edge_v));
break;
case Type::PATH:
new (&path_v) Path(std::move(other.path_v));
break;
}
other.type = Type::NILL;
}
explicit Value(const bool val) : bool_v(val), type(BOOL){};
explicit Value(const int64_t val) : int_v(val), type(INT64){};
explicit Value(const double val) : double_v(val), type(DOUBLE){};
explicit Value(const std::string &val) : string_v(val), type(STRING){};
explicit Value(std::vector<Value> &&val) : list_v(std::move(val)), type(LIST){};
explicit Value(std::map<std::string, Value> &&val) : map_v(std::move(val)), type(MAP){};
explicit Value(const Vertex &val) : vertex_v(val), type(VERTEX){};
explicit Value(const Edge &val) : edge_v(val), type(EDGE){};
explicit Value(const Path &val) : path_v(val), type(PATH){};
Value &operator=(const Value &value) {
if (&value == this) {
return *this;
}
type = value.type;
switch (value.type) {
case NILL:
null_v = {};
break;
case BOOL:
bool_v = value.bool_v;
break;
case INT64:
int_v = value.int_v;
break;
case DOUBLE:
double_v = value.double_v;
break;
case STRING:
string_v = value.string_v;
break;
case LIST:
list_v = value.list_v;
break;
case MAP:
map_v = value.map_v;
break;
case VERTEX:
vertex_v = value.vertex_v;
break;
case EDGE:
edge_v = value.edge_v;
break;
case PATH:
path_v = value.path_v;
break;
}
return *this;
}
friend bool operator==(const Value &lhs, const Value &rhs) {
if (lhs.type != rhs.type) {
return false;
}
switch (lhs.type) {
case NILL:
case Value::Type::Null:
return true;
case BOOL:
case Value::Type::Bool:
return lhs.bool_v == rhs.bool_v;
case INT64:
case Value::Type::Int64:
return lhs.int_v == rhs.int_v;
case DOUBLE:
case Value::Type::Double:
return lhs.double_v == rhs.double_v;
case STRING:
case Value::Type::String:
return lhs.string_v == rhs.string_v;
case LIST:
case Value::Type::List:
return lhs.list_v == rhs.list_v;
case MAP:
case Value::Type::Map:
return lhs.map_v == rhs.map_v;
case VERTEX:
case Value::Type::Vertex:
return lhs.vertex_v == rhs.vertex_v;
case EDGE:
case Value::Type::Edge:
return lhs.edge_v == rhs.edge_v;
case PATH:
case Value::Type::Path:
return true;
}
}
~Value() {
switch (type) {
// destructor for primitive types does nothing
case Type::NILL:
case Type::BOOL:
case Type::INT64:
case Type::DOUBLE:
return;
// destructor for non primitive types since we used placement new
case Type::STRING:
std::destroy_at(&string_v);
return;
case Type::LIST:
std::destroy_at(&list_v);
return;
case Type::MAP:
std::destroy_at(&map_v);
return;
// are these needed to be defined?
case Type::VERTEX:
std::destroy_at(&vertex_v);
return;
case Type::PATH:
std::destroy_at(&path_v);
return;
case Type::EDGE:
std::destroy_at(&edge_v);
default:
return;
}
}
};
struct ValuesMap {
@ -322,19 +377,18 @@ struct OrderBy {
enum class StorageView { OLD = 0, NEW = 1 };
struct ScanVerticesRequest {
using Hlc = memgraph::coordinator::Hlc;
Hlc transaction_id;
VertexId start_id;
std::optional<std::vector<std::string>> props_to_return;
std::optional<std::vector<PropertyId>> props_to_return;
std::optional<std::vector<std::string>> filter_expressions;
std::optional<size_t> batch_limit;
StorageView storage_view;
};
struct ScanResultRow {
Vertex vertex;
// empty is no properties returned
std::map<PropertyId, Value> props;
Value vertex;
// empty() is no properties returned
std::vector<std::pair<PropertyId, Value>> props;
};
struct ScanVerticesResponse {
@ -346,7 +400,6 @@ struct ScanVerticesResponse {
using VertexOrEdgeIds = std::variant<VertexId, EdgeId>;
struct GetPropertiesRequest {
using Hlc = memgraph::coordinator::Hlc;
Hlc transaction_id;
VertexOrEdgeIds vertex_or_edge_ids;
std::vector<PropertyId> property_ids;
@ -369,7 +422,6 @@ struct VertexEdgeId {
std::optional<EdgeId> next_id;
};
using Hlc = memgraph::coordinator::Hlc;
struct ExpandOneRequest {
Hlc transaction_id;
std::vector<VertexId> src_vertices;
@ -414,20 +466,32 @@ struct ExpandOneResponse {
std::vector<ExpandOneResultRow> result;
};
struct UpdateVertexProp {
VertexId primary_key;
std::vector<std::pair<PropertyId, Value>> property_updates;
};
struct UpdateEdgeProp {
Edge edge;
std::vector<std::pair<PropertyId, Value>> property_updates;
};
/*
* Vertices
*/
struct NewVertex {
Label label_ids;
std::vector<Label> label_ids;
PrimaryKey primary_key;
std::map<PropertyId, Value> properties;
std::vector<std::pair<PropertyId, Value>> properties;
};
struct NewVertexLabel {
std::string label;
PrimaryKey primary_key;
std::map<PropertyId, Value> properties;
std::vector<std::pair<PropertyId, Value>> properties;
};
struct CreateVerticesRequest {
using Hlc = memgraph::coordinator::Hlc;
std::string label;
Hlc transaction_id;
std::vector<NewVertex> new_vertices;
@ -437,9 +501,62 @@ struct CreateVerticesResponse {
bool success;
};
struct DeleteVerticesRequest {
enum class DeletionType { DELETE, DETACH_DELETE };
Hlc transaction_id;
std::vector<std::vector<Value>> primary_keys;
DeletionType deletion_type;
};
struct DeleteVerticesResponse {
bool success;
};
struct UpdateVerticesRequest {
Hlc transaction_id;
std::vector<UpdateVertexProp> new_properties;
};
struct UpdateVerticesResponse {
bool success;
};
/*
* Edges
*/
struct CreateEdgesRequest {
Hlc transaction_id;
std::vector<Edge> edges;
};
struct CreateEdgesResponse {
bool success;
};
struct DeleteEdgesRequest {
Hlc transaction_id;
std::vector<Edge> edges;
};
struct DeleteEdgesResponse {
bool success;
};
struct UpdateEdgesRequest {
Hlc transaction_id;
std::vector<UpdateEdgeProp> new_properties;
};
struct UpdateEdgesResponse {
bool success;
};
using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse>;
using WriteRequests = CreateVerticesRequest;
using WriteResponses = CreateVerticesResponse;
} // namespace requests
using WriteRequests = std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest,
CreateEdgesRequest, DeleteEdgesRequest, UpdateEdgesRequest>;
using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesResponse, UpdateVerticesResponse,
CreateEdgesResponse, DeleteEdgesResponse, UpdateEdgesResponse>;
} // namespace memgraph::msgs

35
src/query/v2/shard_request_manager.hpp
View File

@ -36,9 +36,10 @@
#include "query/v2/accessors.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/value_conversions.hpp"
#include "utils/result.hpp"
namespace requests {
namespace memgraph::msgs {
template <typename TStorageClient>
class RsmStorageClientManager {
public:
@ -48,8 +49,8 @@ class RsmStorageClientManager {
RsmStorageClientManager() = default;
RsmStorageClientManager(const RsmStorageClientManager &) = delete;
RsmStorageClientManager(RsmStorageClientManager &&) = delete;
RsmStorageClientManager& operator=(const RsmStorageClientManager &) = delete;
RsmStorageClientManager& operator=(RsmStorageClientManager &&) = delete;
RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
~RsmStorageClientManager() = default;
void AddClient(const LabelId label_id, Shard key, TStorageClient client) {
@ -105,7 +106,7 @@ class ShardRequestManagerInterface {
virtual void StartTransaction() = 0;
virtual std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) = 0;
virtual std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
std::vector<requests::NewVertex> new_vertices) = 0;
std::vector<NewVertex> new_vertices) = 0;
virtual std::vector<ExpandOneResponse> Request(ExecutionState<ExpandOneRequest> &state) = 0;
virtual memgraph::storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
virtual memgraph::storage::v3::LabelId LabelNameToLabelId(const std::string &name) const = 0;
@ -128,7 +129,6 @@ class ShardRequestManager : public ShardRequestManagerInterface {
using ShardMap = memgraph::coordinator::ShardMap;
using CompoundKey = memgraph::coordinator::CompoundKey;
using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
using LabelId = Label::LabelId;
ShardRequestManager(CoordinatorClient coord, memgraph::io::Io<TTransport> &&io)
: coord_cli_(std::move(coord)), io_(std::move(io)) {}
@ -177,7 +177,8 @@ class ShardRequestManager : public ShardRequestManagerInterface {
auto &shard_cache_ref = state.shard_cache;
size_t id = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
auto &storage_client = GetStorageClientForShard(*state.label, state.requests[id].start_id.primary_key);
auto &storage_client = GetStorageClientForShard(
*state.label, storage::conversions::ConvertPropertyVector(state.requests[id].start_id.second));
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
// instead.
auto read_response_result = storage_client.SendReadRequest(state.requests[id]);
@ -193,7 +194,7 @@ class ShardRequestManager : public ShardRequestManagerInterface {
if (!response.next_start_id) {
shard_it = shard_cache_ref.erase(shard_it);
} else {
state.requests[id].start_id.primary_key = response.next_start_id->primary_key;
state.requests[id].start_id.second = response.next_start_id->second;
++shard_it;
}
responses.push_back(std::move(response));
@ -214,9 +215,9 @@ class ShardRequestManager : public ShardRequestManagerInterface {
size_t id = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
// This is fine because all new_vertices of each request end up on the same shard
const Label label = state.requests[id].new_vertices[0].label_ids;
const auto labels = state.requests[id].new_vertices[0].label_ids;
auto primary_key = state.requests[id].new_vertices[0].primary_key;
auto &storage_client = GetStorageClientForShard(*shard_it, label.id);
auto &storage_client = GetStorageClientForShard(*shard_it, labels[0].id);
auto write_response_result = storage_client.SendWriteRequest(state.requests[id]);
// RETRY on timeouts?
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
@ -248,7 +249,7 @@ class ShardRequestManager : public ShardRequestManagerInterface {
size_t id = 0;
// pending_requests on shards
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
const Label primary_label = state.requests[id].src_vertices[0].primary_label;
const Label primary_label = state.requests[id].src_vertices[0].first;
auto &storage_client = GetStorageClientForShard(*shard_it, primary_label.id);
auto read_response_result = storage_client.SendReadRequest(state.requests[id]);
// RETRY on timeouts?
@ -266,8 +267,8 @@ class ShardRequestManager : public ShardRequestManagerInterface {
std::vector<VertexAccessor> PostProcess(std::vector<ScanVerticesResponse> &&responses) const {
std::vector<VertexAccessor> accessors;
for (auto &response : responses) {
for (auto result_row : response.results) {
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props)));
for (auto &result_row : response.results) {
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex.vertex_v), std::move(result_row.props)));
}
}
return accessors;
@ -303,7 +304,8 @@ class ShardRequestManager : public ShardRequestManagerInterface {
std::map<Shard, CreateVerticesRequest> per_shard_request_table;
for (auto &new_vertex : new_vertices) {
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids.id, new_vertex.primary_key);
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
if (!per_shard_request_table.contains(shard)) {
CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
@ -329,7 +331,7 @@ class ShardRequestManager : public ShardRequestManagerInterface {
state.shard_cache.push_back(std::move(shard));
ScanVerticesRequest rqst;
rqst.transaction_id = transaction_id_;
rqst.start_id.primary_key = key;
rqst.start_id.second = storage::conversions::ConvertValueVector(key);
state.requests.push_back(std::move(rqst));
}
state.state = ExecutionState<ScanVerticesRequest>::EXECUTING;
@ -351,7 +353,8 @@ class ShardRequestManager : public ShardRequestManagerInterface {
state.requests.clear();
size_t id = 0;
for (const auto &vertex : top_level_rqst.src_vertices) {
auto shard = shards_map_.GetShardForKey(vertex.primary_label.id, vertex.primary_key);
auto shard =
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
if (!per_shard_request_table.contains(shard)) {
ExpandOneRequest expand_v_rqst = top_level_rqst_template;
per_shard_request_table.insert(std::pair(shard, std::move(expand_v_rqst)));
@ -400,4 +403,4 @@ class ShardRequestManager : public ShardRequestManagerInterface {
memgraph::coordinator::Hlc transaction_id_;
// TODO(kostasrim) Add batch prefetching
};
} // namespace requests
} // namespace memgraph::msgs

2
src/query/v2/stream/streams.cpp
View File

@ -84,7 +84,7 @@ std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformation
template <typename TMessage>
void CallCustomTransformation(const std::string &transformation_name, const std::vector<TMessage> &messages,
mgp_result &result, storage::v3::Storage::Accessor &storage_accessor,
mgp_result &result, storage::v3::Shard::Accessor &storage_accessor,
utils::MemoryResource &memory_resource, const std::string &stream_name) {
DbAccessor db_accessor{&storage_accessor};
{

12
src/query/v2/trigger_context.cpp
View File

@ -264,8 +264,8 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
{
// adapt created_vertices_
auto it = created_vertices_.begin();
for (auto &created_vertex : created_vertices_) {
if (auto maybe_vertex = accessor->FindVertex(created_vertex.object.Gid(), storage::v3::View::OLD); maybe_vertex) {
for ([[maybe_unused]] auto &created_vertex : created_vertices_) {
if (auto maybe_vertex = accessor->FindVertex(kFakeVertexGid); maybe_vertex) {
*it = detail::CreatedObject{*maybe_vertex};
++it;
}
@ -280,7 +280,7 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
const auto adapt_context_with_vertex = [accessor](auto *values) {
auto it = values->begin();
for (auto &value : *values) {
if (auto maybe_vertex = accessor->FindVertex(value.object.Gid(), storage::v3::View::OLD); maybe_vertex) {
if (auto maybe_vertex = accessor->FindVertex(kFakeVertexGid); maybe_vertex) {
*it = std::move(value);
it->object = *maybe_vertex;
++it;
@ -298,7 +298,7 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
// adapt created_edges
auto it = created_edges_.begin();
for (auto &created_edge : created_edges_) {
const auto maybe_from_vertex = accessor->FindVertex(created_edge.object.From().Gid(), storage::v3::View::OLD);
const auto maybe_from_vertex = accessor->FindVertex(kFakeVertexGid);
if (!maybe_from_vertex) {
continue;
}
@ -322,7 +322,7 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
const auto adapt_context_with_edge = [accessor](auto *values) {
auto it = values->begin();
for (const auto &value : *values) {
if (auto maybe_vertex = accessor->FindVertex(value.object.From().Gid(), storage::v3::View::OLD); maybe_vertex) {
if (auto maybe_vertex = accessor->FindVertex(kFakeVertexGid); maybe_vertex) {
auto maybe_out_edges = maybe_vertex->OutEdges(storage::v3::View::OLD);
MG_ASSERT(maybe_out_edges.HasValue());
for (const auto &edge : *maybe_out_edges) {
@ -435,7 +435,7 @@ bool TriggerContext::ShouldEventTrigger(const TriggerEventType event_type) const
void TriggerContextCollector::UpdateLabelMap(const VertexAccessor vertex, const storage::v3::LabelId label_id,
const LabelChange change) {
auto &registry = GetRegistry<VertexAccessor>();
if (!registry.should_register_updated_objects || registry.created_objects.count(vertex.Gid())) {
if (!registry.should_register_updated_objects || registry.created_objects.count(kFakeVertexGid)) {
return;
}

49
src/query/v2/trigger_context.hpp
View File

@ -22,12 +22,16 @@
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/view.hpp"
#include "utils/concepts.hpp"
#include "utils/fnv.hpp"
namespace memgraph::query::v2 {
// TODO(jbajic) Fix triggers
inline constexpr uint64_t kFakeVertexGid{0};
namespace detail {
template <typename T>
concept ObjectAccessor = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor>;
@ -162,8 +166,10 @@ const char *TriggerEventTypeToString(TriggerEventType event_type);
static_assert(std::is_trivially_copy_constructible_v<VertexAccessor>,
"VertexAccessor is not trivially copy constructible, move it where possible and remove this assert");
static_assert(std::is_trivially_copy_constructible_v<EdgeAccessor>,
"EdgeAccessor is not trivially copy constructible, move it where possible and remove this asssert");
// TODO(antaljanosbenjamin): Either satisfy this static_assert or move the edge accessors where it is possible when we
// will support triggers.
// static_assert(std::is_trivially_copy_constructible_v<EdgeAccessor>,
// "EdgeAccessor is not trivially copy constructible, move it where possible and remove this asssert");
// Holds the information necessary for triggers
class TriggerContext {
@ -223,8 +229,13 @@ class TriggerContextCollector {
struct HashPairWithAccessor {
template <detail::ObjectAccessor TAccessor, typename T2>
size_t operator()(const std::pair<TAccessor, T2> &pair) const {
using GidType = decltype(std::declval<TAccessor>().Gid());
return utils::HashCombine<GidType, T2>{}(pair.first.Gid(), pair.second);
// TODO(jbajic) Fix Remove Gid
if constexpr (std::is_same_v<TAccessor, VertexAccessor>) {
return utils::HashCombine<uint64_t, T2>{}(kFakeVertexGid, pair.second);
} else {
using UniqueIdentifierType = decltype(std::declval<TAccessor>().Gid());
return utils::HashCombine<UniqueIdentifierType, T2>{}(pair.first.Gid(), pair.second);
}
}
};
@ -239,10 +250,12 @@ class TriggerContextCollector {
template <detail::ObjectAccessor TAccessor>
struct Registry {
using UniqueIdentifier =
typename std::conditional_t<(std::is_same_v<TAccessor, VertexAccessor>), uint64_t, storage::v3::Gid>;
bool should_register_created_objects{false};
bool should_register_deleted_objects{false};
bool should_register_updated_objects{false}; // Set/removed properties (and labels for vertices)
std::unordered_map<storage::v3::Gid, detail::CreatedObject<TAccessor>> created_objects;
std::unordered_map<UniqueIdentifier, detail::CreatedObject<TAccessor>> created_objects;
std::vector<detail::DeletedObject<TAccessor>> deleted_objects;
// During the transaction, a single property on a single object could be changed multiple times.
// We want to register only the global change, at the end of the transaction. The change consists of
@ -268,7 +281,11 @@ class TriggerContextCollector {
if (!registry.should_register_created_objects) {
return;
}
registry.created_objects.emplace(created_object.Gid(), detail::CreatedObject{created_object});
if constexpr (std::is_same_v<TAccessor, VertexAccessor>) {
registry.created_objects.emplace(kFakeVertexGid, detail::CreatedObject{created_object});
} else {
registry.created_objects.emplace(created_object.Gid(), detail::CreatedObject{created_object});
}
}
template <detail::ObjectAccessor TAccessor>
@ -276,9 +293,8 @@ class TriggerContextCollector {
return GetRegistry<TAccessor>().should_register_deleted_objects;
}
template <detail::ObjectAccessor TAccessor>
void RegisterDeletedObject(const TAccessor &deleted_object) {
auto &registry = GetRegistry<TAccessor>();
void RegisterDeletedObject(const EdgeAccessor &deleted_object) {
auto &registry = GetRegistry<EdgeAccessor>();
if (!registry.should_register_deleted_objects || registry.created_objects.count(deleted_object.Gid())) {
return;
}
@ -286,6 +302,10 @@ class TriggerContextCollector {
registry.deleted_objects.emplace_back(deleted_object);
}
void RegisterDeletedObject(const VertexAccessor &deleted_object) {
// TODO(jbajic) Fix Remove Gid
}
template <detail::ObjectAccessor TAccessor>
bool ShouldRegisterObjectPropertyChange() const {
return GetRegistry<TAccessor>().should_register_updated_objects;
@ -298,9 +318,14 @@ class TriggerContextCollector {
if (!registry.should_register_updated_objects) {
return;
}
if (registry.created_objects.count(object.Gid())) {
return;
if constexpr (std::is_same_v<TAccessor, VertexAccessor>) {
if (registry.created_objects.count(kFakeVertexGid)) {
return;
}
} else {
if (registry.created_objects.count(object.Gid())) {
return;
}
}
if (auto it = registry.property_changes.find({object, key}); it != registry.property_changes.end()) {

10
src/storage/v2/storage.cpp
View File

@ -1110,8 +1110,9 @@ void Storage::Accessor::Abort() {
{
std::unique_lock<utils::SpinLock> engine_guard(storage_->engine_lock_);
uint64_t mark_timestamp = storage_->timestamp_;
// Take garbage_undo_buffers lock while holding the engine lock to make
// sure that entries are sorted by mark timestamp in the list.
// Take garbage_undo_buffers lock while holding the engine lock to make sure that entries are sorted by mark
// timestamp in the list. This is necessary when a transaction is aborting simultaneously with a GC run: both of
// these operations acquire a mark timestamps and then modify the garbage deltas.
storage_->garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) {
// Release engine lock because we don't have to hold it anymore and
// emplace back could take a long time.
@ -1517,8 +1518,9 @@ void Storage::CollectGarbage() {
{
std::unique_lock<utils::SpinLock> guard(engine_lock_);
uint64_t mark_timestamp = timestamp_;
// Take garbage_undo_buffers lock while holding the engine lock to make
// sure that entries are sorted by mark timestamp in the list.
// Take garbage_undo_buffers lock while holding the engine lock to make sure that entries are sorted by mark
// timestamp in the list. This is necessary when a transaction is aborting simultaneously with a GC run: both of
// these operations acquire a mark timestamps and then modify the garbage deltas.
garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) {
// Release engine lock because we don't have to hold it anymore and
// this could take a long time.

6
src/storage/v3/CMakeLists.txt
View File

@ -8,11 +8,15 @@ set(storage_v3_src_files
durability/wal.cpp
edge_accessor.cpp
indices.cpp
key_store.cpp
lexicographically_ordered_vertex.cpp
property_store.cpp
vertex_accessor.cpp
schemas.cpp
schema_validator.cpp
storage.cpp)
shard.cpp
storage.cpp
shard_rsm.cpp)
# #### Replication #####
define_add_lcp(add_lcp_storage lcp_storage_cpp_files generated_lcp_storage_files)

7
src/storage/v3/commit_log.cpp
View File

@ -47,8 +47,6 @@ CommitLog::~CommitLog() {
}
void CommitLog::MarkFinished(uint64_t id) {
std::lock_guard<utils::SpinLock> guard(lock_);
Block *block = FindOrCreateBlock(id);
block->field[(id % kIdsInBlock) / kIdsInField] |= 1ULL << (id % kIdsInField);
if (id == oldest_active_) {
@ -56,10 +54,7 @@ void CommitLog::MarkFinished(uint64_t id) {
}
}
uint64_t CommitLog::OldestActive() {
std::lock_guard<utils::SpinLock> guard(lock_);
return oldest_active_;
}
uint64_t CommitLog::OldestActive() const noexcept { return oldest_active_; }
void CommitLog::UpdateOldestActive() {
while (head_) {

3
src/storage/v3/commit_log.hpp
View File

@ -51,7 +51,7 @@ class CommitLog final {
void MarkFinished(uint64_t id);
/// Retrieve the oldest transaction still not marked as finished.
uint64_t OldestActive();
uint64_t OldestActive() const noexcept;
private:
static constexpr uint64_t kBlockSize = 8192;
@ -72,7 +72,6 @@ class CommitLog final {
uint64_t head_start_{0};
uint64_t next_start_{0};
uint64_t oldest_active_{0};
utils::SpinLock lock_;
utils::Allocator<Block> allocator_;
};

6
src/storage/v3/config.hpp
View File

@ -14,6 +14,7 @@
#include <chrono>
#include <cstdint>
#include <filesystem>
#include "storage/v3/id_types.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/transaction.hpp"
@ -23,9 +24,10 @@ namespace memgraph::storage::v3 {
/// the storage. This class also defines the default behavior.
struct Config {
struct Gc {
enum class Type { NONE, PERIODIC };
// TODO(antaljanosbenjamin): How to handle garbage collection?
enum class Type { NONE };
Type type{Type::PERIODIC};
Type type{Type::NONE};
std::chrono::milliseconds interval{std::chrono::milliseconds(1000)};
} gc;

7
src/storage/v3/constraints.cpp
View File

@ -57,7 +57,6 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
bool deleted{false};
bool has_label{false};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
delta = vertex.delta;
deleted = vertex.deleted;
has_label = VertexHasLabel(vertex, label);
@ -142,7 +141,6 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std::
bool deleted{false};
Delta *delta{nullptr};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
has_label = VertexHasLabel(vertex, label);
deleted = vertex.deleted;
delta = vertex.delta;
@ -280,7 +278,7 @@ void UniqueConstraints::UpdateBeforeCommit(const Vertex *vertex, const Transacti
}
utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> UniqueConstraints::CreateConstraint(
LabelId label, const std::set<PropertyId> &properties, utils::SkipList<Vertex>::Accessor vertices) {
LabelId label, const std::set<PropertyId> &properties, VerticesSkipList::Accessor vertices) {
if (properties.empty()) {
return CreationStatus::EMPTY_PROPERTIES;
}
@ -301,7 +299,8 @@ utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> Uniqu
{
auto acc = constraint->second.access();
for (const Vertex &vertex : vertices) {
for (const auto &lo_vertex : vertices) {
const auto &vertex = lo_vertex.vertex;
if (vertex.deleted || !VertexHasLabel(vertex, label)) {
continue;
}

11
src/storage/v3/constraints.hpp
View File

@ -18,6 +18,7 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
#include "utils/skip_list.hpp"
@ -108,7 +109,7 @@ class UniqueConstraints {
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, CreationStatus> CreateConstraint(LabelId label,
const std::set<PropertyId> &properties,
utils::SkipList<Vertex>::Accessor vertices);
VerticesSkipList::Accessor vertices);
/// Deletes the specified constraint. Returns `DeletionStatus::NOT_FOUND` if
/// there is not such constraint in the storage,
@ -152,12 +153,14 @@ struct Constraints {
///
/// @throw std::bad_alloc
/// @throw std::length_error
inline utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(
Constraints *constraints, LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices) {
inline utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(Constraints *constraints, LabelId label,
PropertyId property,
VerticesSkipList::Accessor vertices) {
if (utils::Contains(constraints->existence_constraints, std::make_pair(label, property))) {
return false;
}
for (const auto &vertex : vertices) {
for (const auto &lgo_vertex : vertices) {
const auto &vertex = lgo_vertex.vertex;
if (!vertex.deleted && VertexHasLabel(vertex, label) && !vertex.properties.HasProperty(property)) {
return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set<PropertyId>{property}};
}

37
src/storage/v3/conversions.hpp
View File

@ -10,6 +10,7 @@
// licenses/APL.txt.
#include "expr/typed_value.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/property_value.hpp"
#pragma once
@ -113,4 +114,40 @@ storage::v3::PropertyValue TypedToPropertyValue(const TTypedValue &value) {
}
throw expr::TypedValueException("Unsupported conversion from TTypedValue to PropertyValue");
}
template <typename TTypedValue>
msgs::Value TypedValueToValue(const TTypedValue &value) {
using Value = msgs::Value;
switch (value.type()) {
case TTypedValue::Type::Null:
return {};
case TTypedValue::Type::Bool:
return Value(value.ValueBool());
case TTypedValue::Type::Int:
return Value(value.ValueInt());
case TTypedValue::Type::Double:
return Value(value.ValueDouble());
case TTypedValue::Type::String:
return Value(std::string(value.ValueString()));
case TTypedValue::Type::List: {
const auto &src = value.ValueList();
std::vector<msgs::Value> dst;
dst.reserve(src.size());
std::transform(src.begin(), src.end(), std::back_inserter(dst),
[](const auto &val) { return TypedValueToValue(val); });
return Value(std::move(dst));
}
case TTypedValue::Type::Map: {
const auto &src = value.ValueMap();
std::map<std::string, Value> dst;
for (const auto &elem : src) {
dst.insert({std::string(elem.first), TypedValueToValue(elem.second)});
}
return Value(std::move(dst));
}
default:
break;
}
throw expr::TypedValueException("Unsupported conversion from TTypedValue to PropertyValue");
}
} // namespace memgraph::storage::v3

27
src/storage/v3/delta.hpp
View File

@ -12,10 +12,12 @@
#pragma once
#include <atomic>
#include <memory>
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex_id.hpp"
#include "utils/logging.hpp"
namespace memgraph::storage::v3 {
@ -173,33 +175,33 @@ struct Delta {
uint64_t command_id)
: action(Action::SET_PROPERTY), timestamp(timestamp), command_id(command_id), property({key, value}) {}
Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
uint64_t command_id)
Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge,
std::atomic<uint64_t> *timestamp, uint64_t command_id)
: action(Action::ADD_IN_EDGE),
timestamp(timestamp),
command_id(command_id),
vertex_edge({edge_type, vertex, edge}) {}
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
uint64_t command_id)
Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge,
std::atomic<uint64_t> *timestamp, uint64_t command_id)
: action(Action::ADD_OUT_EDGE),
timestamp(timestamp),
command_id(command_id),
vertex_edge({edge_type, vertex, edge}) {}
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge,
Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge,
std::atomic<uint64_t> *timestamp, uint64_t command_id)
: action(Action::REMOVE_IN_EDGE),
timestamp(timestamp),
command_id(command_id),
vertex_edge({edge_type, vertex, edge}) {}
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge,
Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge,
std::atomic<uint64_t> *timestamp, uint64_t command_id)
: action(Action::REMOVE_OUT_EDGE),
timestamp(timestamp),
command_id(command_id),
vertex_edge({edge_type, vertex, edge}) {}
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(const Delta &) = delete;
Delta(Delta &&) = delete;
@ -212,11 +214,12 @@ struct Delta {
case Action::RECREATE_OBJECT:
case Action::ADD_LABEL:
case Action::REMOVE_LABEL:
break;
case Action::ADD_IN_EDGE:
case Action::ADD_OUT_EDGE:
case Action::REMOVE_IN_EDGE:
case Action::REMOVE_OUT_EDGE:
break;
std::destroy_at(&vertex_edge.vertex_id);
case Action::SET_PROPERTY:
property.value.~PropertyValue();
break;
@ -239,7 +242,7 @@ struct Delta {
} property;
struct {
EdgeTypeId edge_type;
Vertex *vertex;
VertexId vertex_id;
EdgeRef edge;
} vertex_edge;
};

9
src/storage/v3/durability/durability.cpp
View File

@ -113,7 +113,7 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
// to ensure that the indices and constraints are consistent at the end of the
// recovery process.
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
Constraints *constraints, utils::SkipList<Vertex> *vertices) {
Constraints *constraints, VerticesSkipList *vertices) {
spdlog::info("Recreating indices from metadata.");
// Recover label indices.
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
@ -161,10 +161,9 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
uint64_t *wal_seq_num) {
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, uint64_t *edge_count,
NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints,
Config::Items items, uint64_t *wal_seq_num) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
wal_directory);

10
src/storage/v3/durability/durability.hpp
View File

@ -11,7 +11,6 @@
#pragma once
#include <atomic>
#include <cstdint>
#include <filesystem>
#include <optional>
@ -97,7 +96,7 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
// recovery process.
/// @throw RecoveryFailure
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
Constraints *constraints, utils::SkipList<Vertex> *vertices);
Constraints *constraints, VerticesSkipList *vertices);
/// Recovers data either from a snapshot and/or WAL files.
/// @throw RecoveryFailure
@ -106,9 +105,8 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
uint64_t *wal_seq_num);
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, uint64_t *edge_count,
NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints,
Config::Items items, uint64_t *wal_seq_num);
} // namespace memgraph::storage::v3::durability

159
src/storage/v3/durability/snapshot.cpp
View File

@ -19,7 +19,11 @@
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertex_id.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
@ -90,6 +94,10 @@ namespace memgraph::storage::v3::durability {
// IMPORTANT: When changing snapshot encoding/decoding bump the snapshot/WAL
// version in `version.hpp`.
namespace {
constexpr auto kDummyLabelId = LabelId::FromUint(0);
} // namespace
// Function used to read information about the snapshot file.
SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
// Check magic and version.
@ -157,10 +165,10 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
return info;
}
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items) {
NameIdMapper *name_id_mapper, uint64_t *edge_count, Config::Items items) {
RecoveryInfo ret;
RecoveredIndicesAndConstraints indices_constraints;
@ -224,7 +232,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
};
// Reset current edge count.
edge_count->store(0, std::memory_order_release);
*edge_count = 0;
{
// Recover edges.
@ -298,48 +306,48 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
// Insert vertex.
auto gid = snapshot.ReadUint();
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
if (i > 0 && *gid <= last_vertex_gid) {
throw RecoveryFailure("Invalid snapshot data!");
}
last_vertex_gid = *gid;
spdlog::debug("Recovering vertex {}.", *gid);
auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
// auto gid = snapshot.ReadUint();
// if (!gid) throw RecoveryFailure("Invalid snapshot data!");
// if (i > 0 && *gid <= last_vertex_gid) {
// throw RecoveryFailure("Invalid snapshot data!");
// }
// last_vertex_gid = *gid;
// spdlog::debug("Recovering vertex {}.", *gid);
// auto [it, inserted] = vertex_acc.insert({Vertex{Gid::FromUint(*gid), nullptr}});
// if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
// Recover labels.
spdlog::trace("Recovering labels for vertex {}.", *gid);
{
auto labels_size = snapshot.ReadUint();
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
auto &labels = it->labels;
labels.reserve(*labels_size);
for (uint64_t j = 0; j < *labels_size; ++j) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)),
*gid);
labels.emplace_back(get_label_from_id(*label));
}
}
// spdlog::trace("Recovering labels for vertex {}.", *gid);
// {
// auto labels_size = snapshot.ReadUint();
// if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
// auto &labels = it->vertex.labels;
// labels.reserve(*labels_size);
// for (uint64_t j = 0; j < *labels_size; ++j) {
// auto label = snapshot.ReadUint();
// if (!label) throw RecoveryFailure("Invalid snapshot data!");
// SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)),
// *gid);
// labels.emplace_back(get_label_from_id(*label));
// }
// }
// Recover properties.
spdlog::trace("Recovering properties for vertex {}.", *gid);
{
auto props_size = snapshot.ReadUint();
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
auto &props = it->properties;
for (uint64_t j = 0; j < *props_size; ++j) {
auto key = snapshot.ReadUint();
if (!key) throw RecoveryFailure("Invalid snapshot data!");
auto value = snapshot.ReadPropertyValue();
if (!value) throw RecoveryFailure("Invalid snapshot data!");
SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.",
name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
props.SetProperty(get_property_from_id(*key), *value);
}
}
// spdlog::trace("Recovering properties for vertex {}.", *gid);
// {
// auto props_size = snapshot.ReadUint();
// if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
// auto &props = it->vertex.properties;
// for (uint64_t j = 0; j < *props_size; ++j) {
// auto key = snapshot.ReadUint();
// if (!key) throw RecoveryFailure("Invalid snapshot data!");
// auto value = snapshot.ReadPropertyValue();
// if (!value) throw RecoveryFailure("Invalid snapshot data!");
// SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.",
// name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
// props.SetProperty(get_property_from_id(*key), *value);
// }
// }
// Skip in edges.
{
@ -372,17 +380,18 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover vertices (in/out edges).
spdlog::info("Recovering connectivity.");
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
for (auto &vertex : vertex_acc) {
for (auto &lgo_vertex : vertex_acc) {
auto &vertex = lgo_vertex.vertex;
{
auto marker = snapshot.ReadMarker();
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
}
spdlog::trace("Recovering connectivity for vertex {}.", vertex.gid.AsUint());
// spdlog::trace("Recovering connectivity for vertex {}.", vertex.Gid().AsUint());
// Check vertex.
auto gid = snapshot.ReadUint();
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
if (gid != vertex.gid.AsUint()) throw RecoveryFailure("Invalid snapshot data!");
// auto gid = snapshot.ReadUint();
// if (!gid) throw RecoveryFailure("Invalid snapshot data!");
// if (gid != vertex.Gid().AsUint()) throw RecoveryFailure("Invalid snapshot data!");
// Skip labels.
{
@ -408,7 +417,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover in edges.
{
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
// TODO Fix Gid
spdlog::trace("Recovering inbound edges for vertex {}.", 1);
auto in_size = snapshot.ReadUint();
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.in_edges.reserve(*in_size);
@ -422,7 +432,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
auto edge_type = snapshot.ReadUint();
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
auto from_vertex = vertex_acc.find(Gid::FromUint(*from_gid));
auto from_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*from_gid).AsInt()}});
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid from vertex!");
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
@ -436,15 +446,17 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
edge_ref = EdgeRef(&*edge);
}
}
// TODO Fix Gid
SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->gid.AsUint());
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), 1);
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), VertexId{kDummyLabelId, {}}, edge_ref);
}
}
// Recover out edges.
{
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
// TODO Fix Gid
spdlog::trace("Recovering outbound edges for vertex {}.", 1);
auto out_size = snapshot.ReadUint();
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.out_edges.reserve(*out_size);
@ -458,7 +470,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
auto edge_type = snapshot.ReadUint();
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
auto to_vertex = vertex_acc.find(Gid::FromUint(*to_gid));
auto to_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*to_gid).AsInt()}});
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid to vertex!");
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
@ -472,13 +484,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
edge_ref = EdgeRef(&*edge);
}
}
// TODO Fix Gid
SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->gid.AsUint());
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), 1);
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), VertexId{kDummyLabelId, {}}, edge_ref);
}
// Increment edge count. We only increment the count here because the
// information is duplicated in in_edges.
edge_count->fetch_add(*out_size, std::memory_order_acq_rel);
*edge_count += *out_size;
}
}
spdlog::info("Connectivity is recovered.");
@ -627,9 +640,9 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
const SchemaValidator &schema_validator, const std::string &uuid, const std::string_view epoch_id,
const VertexValidator &vertex_validator, const std::string &uuid, const std::string_view epoch_id,
const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
utils::FileRetainer *file_retainer) {
// Ensure that the storage directory exists.
@ -680,13 +693,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
for (auto &edge : acc) {
// The edge visibility check must be done here manually because we don't
// allow direct access to the edges through the public API.
bool is_visible = true;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge.lock);
is_visible = !edge.deleted;
delta = edge.delta;
}
auto is_visible = !edge.deleted;
auto *delta = edge.delta;
ApplyDeltasForRead(transaction, delta, View::OLD, [&is_visible](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_LABEL:
@ -715,8 +723,14 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
// but that isn't an issue because we won't use that part of the API
// here.
// TODO(jbajic) Fix snapshot with new schema rules
auto ea = EdgeAccessor{edge_ref, EdgeTypeId::FromUint(0UL), nullptr, nullptr, transaction, indices, constraints,
items, schema_validator};
auto ea = EdgeAccessor{edge_ref,
EdgeTypeId::FromUint(0UL),
VertexId{kDummyLabelId, {}},
VertexId{kDummyLabelId, {}},
transaction,
indices,
constraints,
items};
// Get edge data.
auto maybe_props = ea.Properties(View::OLD);
@ -742,9 +756,10 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
{
offset_vertices = snapshot.GetPosition();
auto acc = vertices->access();
for (auto &vertex : acc) {
for (auto &lgo_vertex : acc) {
// The visibility check is implemented for vertices so we use it here.
auto va = VertexAccessor::Create(&vertex, transaction, indices, constraints, items, schema_validator, View::OLD);
auto va = VertexAccessor::Create(&lgo_vertex.vertex, transaction, indices, constraints, items, vertex_validator,
View::OLD);
if (!va) continue;
// Get vertex data.
@ -762,7 +777,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
// Store the vertex.
{
snapshot.WriteMarker(Marker::SECTION_VERTEX);
snapshot.WriteUint(vertex.gid.AsUint());
// TODO Fix Gid
snapshot.WriteUint(1);
const auto &labels = maybe_labels.GetValue();
snapshot.WriteUint(labels.size());
for (const auto &item : labels) {
@ -776,16 +792,17 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
}
const auto &in_edges = maybe_in_edges.GetValue();
snapshot.WriteUint(in_edges.size());
// TODO Disabled serialization for vertices
for (const auto &item : in_edges) {
snapshot.WriteUint(item.Gid().AsUint());
snapshot.WriteUint(item.FromVertex().Gid().AsUint());
snapshot.WriteUint(1);
write_mapping(item.EdgeType());
}
const auto &out_edges = maybe_out_edges.GetValue();
snapshot.WriteUint(out_edges.size());
for (const auto &item : out_edges) {
snapshot.WriteUint(item.Gid().AsUint());
snapshot.WriteUint(item.ToVertex().Gid().AsUint());
snapshot.WriteUint(1);
write_mapping(item.EdgeType());
}
}

6
src/storage/v3/durability/snapshot.hpp
View File

@ -60,15 +60,15 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
/// Function used to load the snapshot data into the storage.
/// @throw RecoveryFailure
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items);
NameIdMapper *name_id_mapper, uint64_t *edge_count, Config::Items items);
/// Function used to create a snapshot using the given transaction.
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
const SchemaValidator &schema_validator, const std::string &uuid, std::string_view epoch_id,
const std::deque<std::pair<std::string, uint64_t>> &epoch_history,

575
src/storage/v3/durability/wal.cpp
View File

@ -17,6 +17,7 @@
#include "storage/v3/durability/version.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
@ -485,54 +486,53 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Ite
// When converting a Delta to a WAL delta the logic is inverted. That is
// because the Delta's represent undo actions and we want to store redo
// actions.
encoder->WriteMarker(Marker::SECTION_DELTA);
encoder->WriteUint(timestamp);
std::lock_guard<utils::SpinLock> guard(vertex.lock);
switch (delta.action) {
case Delta::Action::DELETE_OBJECT:
case Delta::Action::RECREATE_OBJECT: {
encoder->WriteMarker(VertexActionToMarker(delta.action));
encoder->WriteUint(vertex.gid.AsUint());
break;
}
case Delta::Action::SET_PROPERTY: {
encoder->WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY);
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint()));
// The property value is the value that is currently stored in the
// vertex.
// TODO (mferencevic): Mitigate the memory allocation introduced here
// (with the `GetProperty` call). It is the only memory allocation in the
// entire WAL file writing logic.
encoder->WritePropertyValue(vertex.properties.GetProperty(delta.property.key));
break;
}
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL: {
encoder->WriteMarker(VertexActionToMarker(delta.action));
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteString(name_id_mapper->IdToName(delta.label.AsUint()));
break;
}
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_OUT_EDGE: {
encoder->WriteMarker(VertexActionToMarker(delta.action));
if (items.properties_on_edges) {
encoder->WriteUint(delta.vertex_edge.edge.ptr->gid.AsUint());
} else {
encoder->WriteUint(delta.vertex_edge.edge.gid.AsUint());
}
encoder->WriteString(name_id_mapper->IdToName(delta.vertex_edge.edge_type.AsUint()));
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteUint(delta.vertex_edge.vertex->gid.AsUint());
break;
}
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
// These actions are already encoded in the *_OUT_EDGE actions. This
// function should never be called for this type of deltas.
LOG_FATAL("Invalid delta action!");
}
// encoder->WriteMarker(Marker::SECTION_DELTA);
// encoder->WriteUint(timestamp);
// switch (delta.action) {
// case Delta::Action::DELETE_OBJECT:
// case Delta::Action::RECREATE_OBJECT: {
// encoder->WriteMarker(VertexActionToMarker(delta.action));
// encoder->WriteUint(vertex.Gid().AsUint());
// break;
// }
// case Delta::Action::SET_PROPERTY: {
// encoder->WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY);
// encoder->WriteUint(vertex.Gid().AsUint());
// encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint()));
// // The property value is the value that is currently stored in the
// // vertex.
// // TODO (mferencevic): Mitigate the memory allocation introduced here
// // (with the `GetProperty` call). It is the only memory allocation in the
// // entire WAL file writing logic.
// encoder->WritePropertyValue(vertex.properties.GetProperty(delta.property.key));
// break;
// }
// case Delta::Action::ADD_LABEL:
// case Delta::Action::REMOVE_LABEL: {
// encoder->WriteMarker(VertexActionToMarker(delta.action));
// encoder->WriteUint(vertex.Gid().AsUint());
// encoder->WriteString(name_id_mapper->IdToName(delta.label.AsUint()));
// break;
// }
// case Delta::Action::ADD_OUT_EDGE:
// case Delta::Action::REMOVE_OUT_EDGE: {
// encoder->WriteMarker(VertexActionToMarker(delta.action));
// if (items.properties_on_edges) {
// encoder->WriteUint(delta.vertex_edge.edge.ptr->gid.AsUint());
// } else {
// encoder->WriteUint(delta.vertex_edge.edge.gid.AsUint());
// }
// encoder->WriteString(name_id_mapper->IdToName(delta.vertex_edge.edge_type.AsUint()));
// encoder->WriteUint(vertex.Gid().AsUint());
// encoder->WriteUint(delta.vertex_edge.vertex->Gid().AsUint());
// break;
// }
// case Delta::Action::ADD_IN_EDGE:
// case Delta::Action::REMOVE_IN_EDGE:
// // These actions are already encoded in the *_OUT_EDGE actions. This
// // function should never be called for this type of deltas.
// LOG_FATAL("Invalid delta action!");
// }
}
void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta &delta, const Edge &edge,
@ -540,38 +540,37 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta
// When converting a Delta to a WAL delta the logic is inverted. That is
// because the Delta's represent undo actions and we want to store redo
// actions.
encoder->WriteMarker(Marker::SECTION_DELTA);
encoder->WriteUint(timestamp);
std::lock_guard<utils::SpinLock> guard(edge.lock);
switch (delta.action) {
case Delta::Action::SET_PROPERTY: {
encoder->WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY);
encoder->WriteUint(edge.gid.AsUint());
encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint()));
// The property value is the value that is currently stored in the
// edge.
// TODO (mferencevic): Mitigate the memory allocation introduced here
// (with the `GetProperty` call). It is the only memory allocation in the
// entire WAL file writing logic.
encoder->WritePropertyValue(edge.properties.GetProperty(delta.property.key));
break;
}
case Delta::Action::DELETE_OBJECT:
case Delta::Action::RECREATE_OBJECT:
// These actions are already encoded in vertex *_OUT_EDGE actions. Also,
// these deltas don't contain any information about the from vertex, to
// vertex or edge type so they are useless. This function should never
// be called for this type of deltas.
LOG_FATAL("Invalid delta action!");
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_OUT_EDGE:
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
// These deltas shouldn't appear for edges.
LOG_FATAL("Invalid database state!");
}
// encoder->WriteMarker(Marker::SECTION_DELTA);
// encoder->WriteUint(timestamp);
// switch (delta.action) {
// case Delta::Action::SET_PROPERTY: {
// encoder->WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY);
// encoder->WriteUint(edge.gid.AsUint());
// encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint()));
// // The property value is the value that is currently stored in the
// // edge.
// // TODO (mferencevic): Mitigate the memory allocation introduced here
// // (with the `GetProperty` call). It is the only memory allocation in the
// // entire WAL file writing logic.
// encoder->WritePropertyValue(edge.properties.GetProperty(delta.property.key));
// break;
// }
// case Delta::Action::DELETE_OBJECT:
// case Delta::Action::RECREATE_OBJECT:
// // These actions are already encoded in vertex *_OUT_EDGE actions. Also,
// // these deltas don't contain any information about the from vertex, to
// // vertex or edge type so they are useless. This function should never
// // be called for this type of deltas.
// LOG_FATAL("Invalid delta action!");
// case Delta::Action::ADD_LABEL:
// case Delta::Action::REMOVE_LABEL:
// case Delta::Action::ADD_OUT_EDGE:
// case Delta::Action::REMOVE_OUT_EDGE:
// case Delta::Action::ADD_IN_EDGE:
// case Delta::Action::REMOVE_IN_EDGE:
// // These deltas shouldn't appear for edges.
// LOG_FATAL("Invalid database state!");
// }
}
void EncodeTransactionEnd(BaseEncoder *encoder, uint64_t timestamp) {
@ -617,8 +616,8 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Storage
}
RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints,
const std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
const std::optional<uint64_t> last_loaded_timestamp, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, NameIdMapper * /*name_id_mapper*/, uint64_t * /*edge_count*/,
Config::Items items) {
spdlog::info("Trying to load WAL file {}.", path);
RecoveryInfo ret;
@ -644,218 +643,244 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
auto edge_acc = edges->access();
auto vertex_acc = vertices->access();
spdlog::info("WAL file contains {} deltas.", info.num_deltas);
for (uint64_t i = 0; i < info.num_deltas; ++i) {
// Read WAL delta header to find out the delta timestamp.
auto timestamp = ReadWalDeltaHeader(&wal);
// for (uint64_t i = 0; i < info.num_deltas; ++i) {
// // Read WAL delta header to find out the delta timestamp.
// auto timestamp = ReadWalDeltaHeader(&wal);
if (!last_loaded_timestamp || timestamp > *last_loaded_timestamp) {
// This delta should be loaded.
auto delta = ReadWalDeltaData(&wal);
switch (delta.type) {
case WalDeltaData::Type::VERTEX_CREATE: {
auto [vertex, inserted] = vertex_acc.insert(Vertex{delta.vertex_create_delete.gid, nullptr});
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
// if (!last_loaded_timestamp || timestamp > *last_loaded_timestamp) {
// // This delta should be loaded.
// auto delta = ReadWalDeltaData(&wal);
// switch (delta.type) {
// case WalDeltaData::Type::VERTEX_CREATE: {
// auto [vertex, inserted] = vertex_acc.insert({Vertex{delta.vertex_create_delete.gid, nullptr}});
// if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
ret.next_vertex_id = std::max(ret.next_vertex_id, delta.vertex_create_delete.gid.AsUint() + 1);
// ret.next_vertex_id = std::max(ret.next_vertex_id, delta.vertex_create_delete.gid.AsUint() + 1);
break;
}
case WalDeltaData::Type::VERTEX_DELETE: {
auto vertex = vertex_acc.find(delta.vertex_create_delete.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
if (!vertex->in_edges.empty() || !vertex->out_edges.empty())
throw RecoveryFailure("The vertex can't be deleted because it still has edges!");
// break;
// }
// case WalDeltaData::Type::VERTEX_DELETE: {
// auto lgo_vertex_it = vertex_acc.find(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}});
// if (lgo_vertex_it == vertex_acc.end()) {
// throw RecoveryFailure("The vertex doesn't exist!");
// }
// auto &vertex = lgo_vertex_it->vertex;
// if (!vertex.in_edges.empty() || !vertex.out_edges.empty())
// throw RecoveryFailure("The vertex can't be deleted because it still has edges!");
if (!vertex_acc.remove(delta.vertex_create_delete.gid))
throw RecoveryFailure("The vertex must be removed here!");
// if (!vertex_acc.remove(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}}))
// throw RecoveryFailure("The vertex must be removed here!");
break;
}
case WalDeltaData::Type::VERTEX_ADD_LABEL:
case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
auto vertex = vertex_acc.find(delta.vertex_add_remove_label.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
// break;
// }
// case WalDeltaData::Type::VERTEX_ADD_LABEL:
// case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
// auto lgo_vertex_it =
// vertex_acc.find(std::vector{PropertyValue{delta.vertex_add_remove_label.gid.AsInt()}}); if (lgo_vertex_it
// == vertex_acc.end()) {
// throw RecoveryFailure("The vertex doesn't exist!");
// }
// auto &vertex = lgo_vertex_it->vertex;
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.vertex_add_remove_label.label));
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), label_id);
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.vertex_add_remove_label.label));
// auto it = std::find(vertex.labels.begin(), vertex.labels.end(), label_id);
if (delta.type == WalDeltaData::Type::VERTEX_ADD_LABEL) {
if (it != vertex->labels.end()) throw RecoveryFailure("The vertex already has the label!");
vertex->labels.push_back(label_id);
} else {
if (it == vertex->labels.end()) throw RecoveryFailure("The vertex doesn't have the label!");
std::swap(*it, vertex->labels.back());
vertex->labels.pop_back();
}
// if (delta.type == WalDeltaData::Type::VERTEX_ADD_LABEL) {
// if (it != vertex.labels.end()) throw RecoveryFailure("The vertex already has the label!");
// vertex.labels.push_back(label_id);
// } else {
// if (it == vertex.labels.end()) throw RecoveryFailure("The vertex doesn't have the label!");
// std::swap(*it, vertex.labels.back());
// vertex.labels.pop_back();
// }
break;
}
case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
auto vertex = vertex_acc.find(delta.vertex_edge_set_property.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
// break;
// }
// case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
// auto lgo_vertex_it =
// vertex_acc.find(std::vector{PropertyValue{delta.vertex_edge_set_property.gid.AsInt()}}); if (lgo_vertex_it
// == vertex_acc.end()) {
// throw RecoveryFailure("The vertex doesn't exist!");
// }
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property));
auto &property_value = delta.vertex_edge_set_property.value;
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property));
// auto &property_value = delta.vertex_edge_set_property.value;
vertex->properties.SetProperty(property_id, property_value);
// lgo_vertex_it->vertex.properties.SetProperty(property_id, property_value);
break;
}
case WalDeltaData::Type::EDGE_CREATE: {
auto from_vertex = vertex_acc.find(delta.edge_create_delete.from_vertex);
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("The from vertex doesn't exist!");
auto to_vertex = vertex_acc.find(delta.edge_create_delete.to_vertex);
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("The to vertex doesn't exist!");
// break;
// }
// case WalDeltaData::Type::EDGE_CREATE: {
// auto from_lgo_vertex =
// vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}});
// if (from_lgo_vertex == vertex_acc.end()) {
// throw RecoveryFailure("The from vertex doesn't exist!");
// }
// auto to_lgo_vertex =
// vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}}); if (to_lgo_vertex
// == vertex_acc.end()) {
// throw RecoveryFailure("The to vertex doesn't exist!");
// }
// auto &from_vertex = from_lgo_vertex->vertex;
// auto &to_vertex = to_lgo_vertex->vertex;
auto edge_gid = delta.edge_create_delete.gid;
auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
EdgeRef edge_ref(edge_gid);
if (items.properties_on_edges) {
auto [edge, inserted] = edge_acc.insert(Edge{edge_gid, nullptr});
if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
edge_ref = EdgeRef(&*edge);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*to_vertex, edge_ref};
auto it = std::find(from_vertex->out_edges.begin(), from_vertex->out_edges.end(), link);
if (it != from_vertex->out_edges.end()) throw RecoveryFailure("The from vertex already has this edge!");
from_vertex->out_edges.push_back(link);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*from_vertex, edge_ref};
auto it = std::find(to_vertex->in_edges.begin(), to_vertex->in_edges.end(), link);
if (it != to_vertex->in_edges.end()) throw RecoveryFailure("The to vertex already has this edge!");
to_vertex->in_edges.push_back(link);
}
// auto edge_gid = delta.edge_create_delete.gid;
// auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
// EdgeRef edge_ref(edge_gid);
// if (items.properties_on_edges) {
// auto [edge, inserted] = edge_acc.insert(Edge{edge_gid, nullptr});
// if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
// edge_ref = EdgeRef(&*edge);
// }
// {
// std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &to_vertex, edge_ref};
// auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link);
// if (it != from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex already has this edge!");
// from_vertex.out_edges.push_back(link);
// }
// {
// std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &from_vertex, edge_ref};
// auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link);
// if (it != to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex already has this edge!");
// to_vertex.in_edges.push_back(link);
// }
ret.next_edge_id = std::max(ret.next_edge_id, edge_gid.AsUint() + 1);
// ret.next_edge_id = std::max(ret.next_edge_id, edge_gid.AsUint() + 1);
// Increment edge count.
edge_count->fetch_add(1, std::memory_order_acq_rel);
// // Increment edge count.
// *edge_count += 1;
break;
}
case WalDeltaData::Type::EDGE_DELETE: {
auto from_vertex = vertex_acc.find(delta.edge_create_delete.from_vertex);
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("The from vertex doesn't exist!");
auto to_vertex = vertex_acc.find(delta.edge_create_delete.to_vertex);
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("The to vertex doesn't exist!");
// break;
// }
// case WalDeltaData::Type::EDGE_DELETE: {
// auto from_lgo_vertex =
// vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}});
// if (from_lgo_vertex == vertex_acc.end()) {
// throw RecoveryFailure("The from vertex doesn't exist!");
// }
// auto to_lgo_vertex =
// vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}}); if (to_lgo_vertex
// == vertex_acc.end()) {
// throw RecoveryFailure("The to vertex doesn't exist!");
// }
// auto &from_vertex = from_lgo_vertex->vertex;
// auto &to_vertex = to_lgo_vertex->vertex;
auto edge_gid = delta.edge_create_delete.gid;
auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
EdgeRef edge_ref(edge_gid);
if (items.properties_on_edges) {
auto edge = edge_acc.find(edge_gid);
if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!");
edge_ref = EdgeRef(&*edge);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*to_vertex, edge_ref};
auto it = std::find(from_vertex->out_edges.begin(), from_vertex->out_edges.end(), link);
if (it == from_vertex->out_edges.end()) throw RecoveryFailure("The from vertex doesn't have this edge!");
std::swap(*it, from_vertex->out_edges.back());
from_vertex->out_edges.pop_back();
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*from_vertex, edge_ref};
auto it = std::find(to_vertex->in_edges.begin(), to_vertex->in_edges.end(), link);
if (it == to_vertex->in_edges.end()) throw RecoveryFailure("The to vertex doesn't have this edge!");
std::swap(*it, to_vertex->in_edges.back());
to_vertex->in_edges.pop_back();
}
if (items.properties_on_edges) {
if (!edge_acc.remove(edge_gid)) throw RecoveryFailure("The edge must be removed here!");
}
// auto edge_gid = delta.edge_create_delete.gid;
// auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
// EdgeRef edge_ref(edge_gid);
// if (items.properties_on_edges) {
// auto edge = edge_acc.find(edge_gid);
// if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!");
// edge_ref = EdgeRef(&*edge);
// }
// {
// std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &to_vertex, edge_ref};
// auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link);
// if (it == from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex doesn't have this edge!");
// std::swap(*it, from_vertex.out_edges.back());
// from_vertex.out_edges.pop_back();
// }
// {
// std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &from_vertex, edge_ref};
// auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link);
// if (it == to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex doesn't have this edge!");
// std::swap(*it, to_vertex.in_edges.back());
// to_vertex.in_edges.pop_back();
// }
// if (items.properties_on_edges) {
// if (!edge_acc.remove(edge_gid)) throw RecoveryFailure("The edge must be removed here!");
// }
// Decrement edge count.
edge_count->fetch_add(-1, std::memory_order_acq_rel);
// // Decrement edge count.
// *edge_count += -1;
break;
}
case WalDeltaData::Type::EDGE_SET_PROPERTY: {
if (!items.properties_on_edges)
throw RecoveryFailure(
"The WAL has properties on edges, but the storage is "
"configured without properties on edges!");
auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!");
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property));
auto &property_value = delta.vertex_edge_set_property.value;
edge->properties.SetProperty(property_id, property_value);
break;
}
case WalDeltaData::Type::TRANSACTION_END:
break;
case WalDeltaData::Type::LABEL_INDEX_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label));
AddRecoveredIndexConstraint(&indices_constraints->indices.label, label_id, "The label index already exists!");
break;
}
case WalDeltaData::Type::LABEL_INDEX_DROP: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label));
RemoveRecoveredIndexConstraint(&indices_constraints->indices.label, label_id,
"The label index doesn't exist!");
break;
}
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
AddRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id},
"The label property index already exists!");
break;
}
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
RemoveRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id},
"The label property index doesn't exist!");
break;
}
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
AddRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id},
"The existence constraint already exists!");
break;
}
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
RemoveRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id},
"The existence constraint doesn't exist!");
break;
}
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label));
std::set<PropertyId> property_ids;
for (const auto &prop : delta.operation_label_properties.properties) {
property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop)));
}
AddRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids},
"The unique constraint already exists!");
break;
}
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label));
std::set<PropertyId> property_ids;
for (const auto &prop : delta.operation_label_properties.properties) {
property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop)));
}
RemoveRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids},
"The unique constraint doesn't exist!");
break;
}
}
ret.next_timestamp = std::max(ret.next_timestamp, timestamp + 1);
++deltas_applied;
} else {
// This delta should be skipped.
SkipWalDeltaData(&wal);
}
}
// break;
// }
// case WalDeltaData::Type::EDGE_SET_PROPERTY: {
// if (!items.properties_on_edges)
// throw RecoveryFailure(
// "The WAL has properties on edges, but the storage is "
// "configured without properties on edges!");
// auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
// if (edge == edge_acc.end()) throw RecoveryFailure("The edge doesn't exist!");
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property));
// auto &property_value = delta.vertex_edge_set_property.value;
// edge->properties.SetProperty(property_id, property_value);
// break;
// }
// case WalDeltaData::Type::TRANSACTION_END:
// break;
// case WalDeltaData::Type::LABEL_INDEX_CREATE: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label));
// AddRecoveredIndexConstraint(&indices_constraints->indices.label, label_id, "The label index already
// exists!"); break;
// }
// case WalDeltaData::Type::LABEL_INDEX_DROP: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label));
// RemoveRecoveredIndexConstraint(&indices_constraints->indices.label, label_id,
// "The label index doesn't exist!");
// break;
// }
// case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
// AddRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id},
// "The label property index already exists!");
// break;
// }
// case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
// RemoveRecoveredIndexConstraint(&indices_constraints->indices.label_property, {label_id, property_id},
// "The label property index doesn't exist!");
// break;
// }
// case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
// AddRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id},
// "The existence constraint already exists!");
// break;
// }
// case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
// auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));
// RemoveRecoveredIndexConstraint(&indices_constraints->constraints.existence, {label_id, property_id},
// "The existence constraint doesn't exist!");
// break;
// }
// case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label));
// std::set<PropertyId> property_ids;
// for (const auto &prop : delta.operation_label_properties.properties) {
// property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop)));
// }
// AddRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids},
// "The unique constraint already exists!");
// break;
// }
// case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
// auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_properties.label));
// std::set<PropertyId> property_ids;
// for (const auto &prop : delta.operation_label_properties.properties) {
// property_ids.insert(PropertyId::FromUint(name_id_mapper->NameToId(prop)));
// }
// RemoveRecoveredIndexConstraint(&indices_constraints->constraints.unique, {label_id, property_ids},
// "The unique constraint doesn't exist!");
// break;
// }
// }
// ret.next_timestamp = std::max(ret.next_timestamp, timestamp + 1);
// ++deltas_applied;
// } else {
// // This delta should be skipped.
// SkipWalDeltaData(&wal);
// }
// }
spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied,
info.num_deltas - deltas_applied);
// spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied,
// info.num_deltas - deltas_applied);
return ret;
}

5
src/storage/v3/durability/wal.hpp
View File

@ -25,6 +25,7 @@
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/skip_list.hpp"
@ -189,8 +190,8 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Storage
/// Function used to load the WAL data into the storage.
/// @throw RecoveryFailure
RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints,
std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
std::optional<uint64_t> last_loaded_timestamp, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, uint64_t *edge_count,
Config::Items items);
/// WalFile class used to append deltas and operations to the WAL file.

1
src/storage/v3/edge.hpp
View File

@ -33,7 +33,6 @@ struct Edge {
PropertyStore properties;
mutable utils::SpinLock lock;
bool deleted;
// uint8_t PAD;
// uint16_t PAD;

54
src/storage/v3/edge_accessor.cpp
View File

@ -22,14 +22,10 @@
namespace memgraph::storage::v3 {
bool EdgeAccessor::IsVisible(const View view) const {
bool deleted = true;
bool exists = true;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
deleted = edge_.ptr->deleted;
delta = edge_.ptr->delta;
}
auto deleted = edge_.ptr->deleted;
auto exists = true;
auto *delta = edge_.ptr->delta;
ApplyDeltasForRead(transaction_, delta, view, [&](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_LABEL:
@ -54,20 +50,14 @@ bool EdgeAccessor::IsVisible(const View view) const {
return exists && (for_deleted_ || !deleted);
}
VertexAccessor EdgeAccessor::FromVertex() const {
return {from_vertex_, transaction_, indices_, constraints_, config_, *schema_validator_};
}
const VertexId &EdgeAccessor::FromVertex() const { return from_vertex_; }
VertexAccessor EdgeAccessor::ToVertex() const {
return {to_vertex_, transaction_, indices_, constraints_, config_, *schema_validator_};
}
const VertexId &EdgeAccessor::ToVertex() const { return to_vertex_; }
Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
@ -88,8 +78,6 @@ Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const Prope
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
@ -106,16 +94,11 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
if (!config_.properties_on_edges) return PropertyValue();
bool exists = true;
bool deleted = false;
PropertyValue value;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
deleted = edge_.ptr->deleted;
value = edge_.ptr->properties.GetProperty(property);
delta = edge_.ptr->delta;
}
auto exists = true;
auto deleted = edge_.ptr->deleted;
auto value = edge_.ptr->properties.GetProperty(property);
auto *delta = edge_.ptr->delta;
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &value, property](const Delta &delta) {
switch (delta.action) {
case Delta::Action::SET_PROPERTY: {
@ -148,16 +131,11 @@ Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view)
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
if (!config_.properties_on_edges) return std::map<PropertyId, PropertyValue>{};
bool exists = true;
bool deleted = false;
std::map<PropertyId, PropertyValue> properties;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
deleted = edge_.ptr->deleted;
properties = edge_.ptr->properties.Properties();
delta = edge_.ptr->delta;
}
auto exists = true;
auto deleted = edge_.ptr->deleted;
auto properties = edge_.ptr->properties.Properties();
auto *delta = edge_.ptr->delta;
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
switch (delta.action) {
case Delta::Action::SET_PROPERTY: {

22
src/storage/v3/edge_accessor.hpp
View File

@ -17,6 +17,7 @@
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/transaction.hpp"
@ -31,29 +32,27 @@ struct Constraints;
class EdgeAccessor final {
private:
friend class Storage;
friend class Shard;
public:
EdgeAccessor(EdgeRef edge, EdgeTypeId edge_type, Vertex *from_vertex, Vertex *to_vertex, Transaction *transaction,
Indices *indices, Constraints *constraints, Config::Items config,
const SchemaValidator &schema_validator, bool for_deleted = false)
EdgeAccessor(EdgeRef edge, EdgeTypeId edge_type, VertexId from_vertex, VertexId to_vertex, Transaction *transaction,
Indices *indices, Constraints *constraints, Config::Items config, bool for_deleted = false)
: edge_(edge),
edge_type_(edge_type),
from_vertex_(from_vertex),
to_vertex_(to_vertex),
from_vertex_(std::move(from_vertex)),
to_vertex_(std::move(to_vertex)),
transaction_(transaction),
indices_(indices),
constraints_(constraints),
config_(config),
schema_validator_{&schema_validator},
for_deleted_(for_deleted) {}
/// @return true if the object is visible from the current transaction
bool IsVisible(View view) const;
VertexAccessor FromVertex() const;
const VertexId &FromVertex() const;
VertexAccessor ToVertex() const;
const VertexId &ToVertex() const;
EdgeTypeId EdgeType() const { return edge_type_; }
@ -88,13 +87,12 @@ class EdgeAccessor final {
private:
EdgeRef edge_;
EdgeTypeId edge_type_;
Vertex *from_vertex_;
Vertex *to_vertex_;
VertexId from_vertex_;
VertexId to_vertex_;
Transaction *transaction_;
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
// if the accessor was created for a deleted edge.
// Accessor behaves differently for some methods based on this

56
src/storage/v3/id_types.hpp
View File

@ -11,6 +11,8 @@
#pragma once
#include <bit>
#include <cstdint>
#include <functional>
#include <type_traits>
@ -19,30 +21,36 @@
namespace memgraph::storage::v3 {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define STORAGE_DEFINE_ID_TYPE(name) \
class name final { \
private: \
explicit name(uint64_t id) : id_(id) {} \
\
public: \
/* Default constructor to allow serialization or preallocation. */ \
name() = default; \
\
static name FromUint(uint64_t id) { return (name){id}; } \
static name FromInt(int64_t id) { return (name){utils::MemcpyCast<uint64_t>(id)}; } \
uint64_t AsUint() const { return id_; } \
int64_t AsInt() const { return utils::MemcpyCast<int64_t>(id_); } \
\
private: \
uint64_t id_; \
}; \
static_assert(std::is_trivially_copyable<name>::value, "storage::" #name " must be trivially copyable!"); \
inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \
inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \
inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \
inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \
inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \
inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); }
#define STORAGE_DEFINE_ID_TYPE(name) \
class name final { \
private: \
constexpr explicit name(uint64_t id) : id_(id) {} \
\
public: \
/* Default constructor to allow serialization or preallocation. */ \
constexpr name() = default; \
\
constexpr static name FromUint(uint64_t id) { return (name){id}; } \
constexpr static name FromInt(int64_t id) { return (name){std::bit_cast<uint64_t>(id)}; } \
constexpr uint64_t AsUint() const { return id_; } \
constexpr int64_t AsInt() const { return std::bit_cast<int64_t>(id_); } \
\
private: \
uint64_t id_; \
}; \
static_assert(std::is_trivially_copyable<name>::value, "storage::" #name " must be trivially copyable!"); \
constexpr inline bool operator==(const name &first, const name &second) { \
return first.AsUint() == second.AsUint(); \
} \
constexpr inline bool operator!=(const name &first, const name &second) { \
return first.AsUint() != second.AsUint(); \
} \
constexpr inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \
constexpr inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \
constexpr inline bool operator<=(const name &first, const name &second) { \
return first.AsUint() <= second.AsUint(); \
} \
constexpr inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); }
STORAGE_DEFINE_ID_TYPE(Gid);
STORAGE_DEFINE_ID_TYPE(LabelId);

33
src/storage/v3/indices.cpp
View File

@ -13,9 +13,10 @@
#include <limits>
#include "storage/v3/id_types.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "utils/bound.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
@ -52,7 +53,6 @@ bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t timestamp)
bool deleted{false};
const Delta *delta{nullptr};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
has_label = utils::Contains(vertex.labels, label);
deleted = vertex.deleted;
delta = vertex.delta;
@ -105,7 +105,6 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, PropertyId
bool deleted{false};
const Delta *delta{nullptr};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
has_label = utils::Contains(vertex.labels, label);
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
deleted = vertex.deleted;
@ -164,7 +163,6 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
bool has_label{false};
const Delta *delta{nullptr};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
deleted = vertex.deleted;
has_label = utils::Contains(vertex.labels, label);
delta = vertex.delta;
@ -216,7 +214,6 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
bool current_value_equal_to_value = value.IsNull();
const Delta *delta{nullptr};
{
std::lock_guard<utils::SpinLock> guard(vertex.lock);
deleted = vertex.deleted;
has_label = utils::Contains(vertex.labels, label);
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
@ -272,7 +269,7 @@ void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transacti
acc.insert(Entry{vertex, tx.start_timestamp});
}
bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices) {
bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
if (!emplaced) {
@ -281,7 +278,8 @@ bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor ve
}
try {
auto acc = it->second.access();
for (Vertex &vertex : vertices) {
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}
@ -329,7 +327,7 @@ void LabelIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) {
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->schema_validator_),
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
current_vertex_(nullptr) {
AdvanceUntilValid();
}
@ -348,7 +346,7 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
if (CurrentVersionHasLabel(*index_iterator_->vertex, self_->label_, self_->transaction_, self_->view_)) {
current_vertex_ = index_iterator_->vertex;
current_vertex_accessor_ = VertexAccessor{current_vertex_, self_->transaction_, self_->indices_,
self_->constraints_, self_->config_, *self_->schema_validator_};
self_->constraints_, self_->config_, *self_->vertex_validator_};
break;
}
}
@ -356,7 +354,7 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view,
Transaction *transaction, Indices *indices, Constraints *constraints,
Config::Items config, const SchemaValidator &schema_validator)
Config::Items config, const VertexValidator &vertex_validator)
: index_accessor_(std::move(index_accessor)),
label_(label),
view_(view),
@ -364,7 +362,7 @@ LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor,
indices_(indices),
constraints_(constraints),
config_(config),
schema_validator_(&schema_validator) {}
vertex_validator_(&vertex_validator) {}
void LabelIndex::RunGC() {
for (auto &index_entry : index_) {
@ -419,7 +417,7 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
}
}
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices) {
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] =
index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
@ -429,7 +427,8 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::
}
try {
auto acc = it->second.access();
for (Vertex &vertex : vertices) {
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}
@ -481,7 +480,7 @@ void LabelPropertyIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_time
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->schema_validator_),
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
current_vertex_(nullptr) {
AdvanceUntilValid();
}
@ -521,7 +520,7 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
index_iterator_->value, self_->transaction_, self_->view_)) {
current_vertex_ = index_iterator_->vertex;
current_vertex_accessor_ = VertexAccessor(current_vertex_, self_->transaction_, self_->indices_,
self_->constraints_, self_->config_, *self_->schema_validator_);
self_->constraints_, self_->config_, *self_->vertex_validator_);
break;
}
}
@ -544,7 +543,7 @@ LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_ac
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
Transaction *transaction, Indices *indices, Constraints *constraints,
Config::Items config, const SchemaValidator &schema_validator)
Config::Items config, const VertexValidator &vertex_validator)
: index_accessor_(std::move(index_accessor)),
label_(label),
property_(property),
@ -555,7 +554,7 @@ LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_ac
indices_(indices),
constraints_(constraints),
config_(config),
schema_validator_(&schema_validator) {
vertex_validator_(&vertex_validator) {
// We have to fix the bounds that the user provided to us. If the user
// provided only one bound we should make sure that only values of that type
// are returned by the iterator. We ensure this by supplying either an

42
src/storage/v3/indices.hpp
View File

@ -18,9 +18,9 @@
#include "storage/v3/config.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/bound.hpp"
#include "utils/logging.hpp"
#include "utils/skip_list.hpp"
@ -53,14 +53,14 @@ class LabelIndex {
};
public:
LabelIndex(Indices *indices, Constraints *constraints, Config::Items config, const SchemaValidator &schema_validator)
: indices_(indices), constraints_(constraints), config_(config), schema_validator_{&schema_validator} {}
LabelIndex(Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator)
: indices_(indices), constraints_(constraints), config_(config), vertex_validator_{&vertex_validator} {}
/// @throw std::bad_alloc
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices);
bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
/// Returns false if there was no index to drop
bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
@ -74,7 +74,7 @@ class LabelIndex {
class Iterable {
public:
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view, Transaction *transaction,
Indices *indices, Constraints *constraints, Config::Items config, const SchemaValidator &schema_validator);
Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
@ -107,14 +107,14 @@ class LabelIndex {
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
const VertexValidator *vertex_validator_;
};
/// Returns an self with vertices visible from the given transaction.
Iterable Vertices(LabelId label, View view, Transaction *transaction) {
auto it = index_.find(label);
MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint());
return {it->second.access(), label, view, transaction, indices_, constraints_, config_, *schema_validator_};
return {it->second.access(), label, view, transaction, indices_, constraints_, config_, *vertex_validator_};
}
int64_t ApproximateVertexCount(LabelId label) {
@ -132,7 +132,7 @@ class LabelIndex {
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
const VertexValidator *vertex_validator_;
};
class LabelPropertyIndex {
@ -151,8 +151,8 @@ class LabelPropertyIndex {
public:
LabelPropertyIndex(Indices *indices, Constraints *constraints, Config::Items config,
const SchemaValidator &schema_validator)
: indices_(indices), constraints_(constraints), config_(config), schema_validator_{&schema_validator} {}
const VertexValidator &vertex_validator)
: indices_(indices), constraints_(constraints), config_(config), vertex_validator_{&vertex_validator} {}
/// @throw std::bad_alloc
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
@ -161,7 +161,7 @@ class LabelPropertyIndex {
void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices);
bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
@ -176,7 +176,7 @@ class LabelPropertyIndex {
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
Indices *indices, Constraints *constraints, Config::Items config, const SchemaValidator &schema_validator);
Indices *indices, Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
@ -213,17 +213,17 @@ class LabelPropertyIndex {
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
const VertexValidator *vertex_validator_;
};
Iterable Vertices(LabelId label, PropertyId property, const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
const SchemaValidator &schema_validator_) {
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
Transaction *transaction) {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
property.AsUint());
return {it->second.access(), label, property, lower_bound, upper_bound, view,
transaction, indices_, constraints_, config_, schema_validator_};
return {it->second.access(), label, property, lower_bound, upper_bound, view,
transaction, indices_, constraints_, config_, *vertex_validator_};
}
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
@ -252,13 +252,13 @@ class LabelPropertyIndex {
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
const VertexValidator *vertex_validator_;
};
struct Indices {
Indices(Constraints *constraints, Config::Items config, const SchemaValidator &schema_validator)
: label_index(this, constraints, config, schema_validator),
label_property_index(this, constraints, config, schema_validator) {}
Indices(Constraints *constraints, Config::Items config, const VertexValidator &vertex_validator)
: label_index(this, constraints, config, vertex_validator),
label_property_index(this, constraints, config, vertex_validator) {}
// Disable copy and move because members hold pointer to `this`.
Indices(const Indices &) = delete;

43
src/storage/v3/key_store.cpp Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <iterator>
#include <ranges>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
KeyStore::KeyStore(const PrimaryKey &key_values) {
for (auto i = 0; i < key_values.size(); ++i) {
MG_ASSERT(!key_values[i].IsNull());
store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
}
}
PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
PropertyValue KeyStore::GetKey(const PropertyId property_id) const { return store_.GetProperty(property_id); }
PrimaryKey KeyStore::Keys() const {
auto keys_map = store_.Properties();
PrimaryKey keys;
keys.reserve(keys_map.size());
std::ranges::transform(
keys_map, std::back_inserter(keys),
[](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
return keys;
}
} // namespace memgraph::storage::v3

64
src/storage/v3/key_store.hpp Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <algorithm>
#include <compare>
#include <functional>
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
// Primary key is a collection of primary properties.
using PrimaryKey = std::vector<PropertyValue>;
class KeyStore {
public:
explicit KeyStore(const PrimaryKey &key_values);
KeyStore(const KeyStore &) = delete;
KeyStore(KeyStore &&other) noexcept = default;
KeyStore &operator=(const KeyStore &) = delete;
KeyStore &operator=(KeyStore &&other) noexcept = default;
~KeyStore() = default;
PropertyValue GetKey(size_t index) const;
PropertyValue GetKey(PropertyId property) const;
PrimaryKey Keys() const;
friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
// TODO(antaljanosbenjamin): also compare the schema
return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
return std::ranges::equal(lhs.Keys(), rhs.Keys());
}
friend bool operator<(const KeyStore &lhs, const PrimaryKey &rhs) {
// TODO(antaljanosbenjamin): also compare the schema
return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const PrimaryKey &rhs) { return std::ranges::equal(lhs.Keys(), rhs); }
private:
PropertyStore store_;
};
} // namespace memgraph::storage::v3

14
src/storage/v3/lexicographically_ordered_vertex.cpp Normal file
View File

@ -0,0 +1,14 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage/v3/lexicographically_ordered_vertex.hpp"
namespace memgraph::storage::v3 {} // namespace memgraph::storage::v3

42
src/storage/v3/lexicographically_ordered_vertex.hpp Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <concepts>
#include <type_traits>
#include "storage/v3/vertex.hpp"
#include "utils/concepts.hpp"
namespace memgraph::storage::v3 {
struct LexicographicallyOrderedVertex {
Vertex vertex;
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys == rhs.vertex.keys;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys < rhs.vertex.keys;
}
// TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys == rhs;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys < rhs;
}
};
} // namespace memgraph::storage::v3

3
src/storage/v3/mvcc.hpp
View File

@ -114,6 +114,9 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
// concurrently (as well as other execution threads).
// 1. We need to set the next delta of the new delta to the existing delta.
// TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in
// `Shard::Accessor::CreateEdge`.
// NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
delta->next.store(object->delta, std::memory_order_release);
// 2. We need to set the previous delta of the new delta to the object.
delta->prev.Set(object);

131
src/storage/v3/replication/replication_client.cpp
View File

@ -30,10 +30,10 @@ template <typename>
} // namespace
////// ReplicationClient //////
Storage::ReplicationClient::ReplicationClient(std::string name, Storage *storage, const io::network::Endpoint &endpoint,
const replication::ReplicationMode mode,
const replication::ReplicationClientConfig &config)
: name_(std::move(name)), storage_(storage), mode_(mode) {
Shard::ReplicationClient::ReplicationClient(std::string name, Shard *shard, const io::network::Endpoint &endpoint,
const replication::ReplicationMode mode,
const replication::ReplicationClientConfig &config)
: name_(std::move(name)), shard_(shard), mode_(mode) {
if (config.ssl) {
rpc_context_.emplace(config.ssl->key_file, config.ssl->cert_file);
} else {
@ -54,14 +54,14 @@ Storage::ReplicationClient::ReplicationClient(std::string name, Storage *storage
}
}
void Storage::ReplicationClient::TryInitializeClientAsync() {
void Shard::ReplicationClient::TryInitializeClientAsync() {
thread_pool_.AddTask([this] {
rpc_client_->Abort();
this->TryInitializeClientSync();
});
}
void Storage::ReplicationClient::FrequentCheck() {
void Shard::ReplicationClient::FrequentCheck() {
const auto is_success = std::invoke([this]() {
try {
auto stream{rpc_client_->Stream<replication::FrequentHeartbeatRpc>()};
@ -87,22 +87,15 @@ void Storage::ReplicationClient::FrequentCheck() {
}
/// @throws rpc::RpcFailedException
void Storage::ReplicationClient::InitializeClient() {
void Shard::ReplicationClient::InitializeClient() {
uint64_t current_commit_timestamp{kTimestampInitialId};
std::optional<std::string> epoch_id;
{
// epoch_id_ can be changed if we don't take this lock
std::unique_lock engine_guard(storage_->engine_lock_);
epoch_id.emplace(storage_->epoch_id_);
}
auto stream{rpc_client_->Stream<replication::HeartbeatRpc>(storage_->last_commit_timestamp_, std::move(*epoch_id))};
auto stream{rpc_client_->Stream<replication::HeartbeatRpc>(shard_->last_commit_timestamp_, shard_->epoch_id_)};
const auto response = stream.AwaitResponse();
std::optional<uint64_t> branching_point;
if (response.epoch_id != storage_->epoch_id_ && response.current_commit_timestamp != kTimestampInitialId) {
const auto &epoch_history = storage_->epoch_history_;
if (response.epoch_id != shard_->epoch_id_ && response.current_commit_timestamp != kTimestampInitialId) {
const auto &epoch_history = shard_->epoch_history_;
const auto epoch_info_iter =
std::find_if(epoch_history.crbegin(), epoch_history.crend(),
[&](const auto &epoch_info) { return epoch_info.first == response.epoch_id; });
@ -122,8 +115,8 @@ void Storage::ReplicationClient::InitializeClient() {
current_commit_timestamp = response.current_commit_timestamp;
spdlog::trace("Current timestamp on replica: {}", current_commit_timestamp);
spdlog::trace("Current timestamp on main: {}", storage_->last_commit_timestamp_.load());
if (current_commit_timestamp == storage_->last_commit_timestamp_.load()) {
spdlog::trace("Current timestamp on main: {}", shard_->last_commit_timestamp_);
if (current_commit_timestamp == shard_->last_commit_timestamp_) {
spdlog::debug("Replica '{}' up to date", name_);
std::unique_lock client_guard{client_lock_};
replica_state_.store(replication::ReplicaState::READY);
@ -137,7 +130,7 @@ void Storage::ReplicationClient::InitializeClient() {
}
}
void Storage::ReplicationClient::TryInitializeClientSync() {
void Shard::ReplicationClient::TryInitializeClientSync() {
try {
InitializeClient();
} catch (const rpc::RpcFailedException &) {
@ -148,19 +141,19 @@ void Storage::ReplicationClient::TryInitializeClientSync() {
}
}
void Storage::ReplicationClient::HandleRpcFailure() {
void Shard::ReplicationClient::HandleRpcFailure() {
spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication"));
TryInitializeClientAsync();
}
replication::SnapshotRes Storage::ReplicationClient::TransferSnapshot(const std::filesystem::path &path) {
replication::SnapshotRes Shard::ReplicationClient::TransferSnapshot(const std::filesystem::path &path) {
auto stream{rpc_client_->Stream<replication::SnapshotRpc>()};
replication::Encoder encoder(stream.GetBuilder());
encoder.WriteFile(path);
return stream.AwaitResponse();
}
replication::WalFilesRes Storage::ReplicationClient::TransferWalFiles(
replication::WalFilesRes Shard::ReplicationClient::TransferWalFiles(
const std::vector<std::filesystem::path> &wal_files) {
MG_ASSERT(!wal_files.empty(), "Wal files list is empty!");
auto stream{rpc_client_->Stream<replication::WalFilesRpc>(wal_files.size())};
@ -173,7 +166,7 @@ replication::WalFilesRes Storage::ReplicationClient::TransferWalFiles(
return stream.AwaitResponse();
}
void Storage::ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) {
void Shard::ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) {
std::unique_lock guard(client_lock_);
const auto status = replica_state_.load();
switch (status) {
@ -197,7 +190,7 @@ void Storage::ReplicationClient::StartTransactionReplication(const uint64_t curr
case replication::ReplicaState::READY:
MG_ASSERT(!replica_stream_);
try {
replica_stream_.emplace(ReplicaStream{this, storage_->last_commit_timestamp_.load(), current_wal_seq_num});
replica_stream_.emplace(ReplicaStream{this, shard_->last_commit_timestamp_, current_wal_seq_num});
replica_state_.store(replication::ReplicaState::REPLICATING);
} catch (const rpc::RpcFailedException &) {
replica_state_.store(replication::ReplicaState::INVALID);
@ -207,7 +200,7 @@ void Storage::ReplicationClient::StartTransactionReplication(const uint64_t curr
}
}
void Storage::ReplicationClient::IfStreamingTransaction(const std::function<void(ReplicaStream &handler)> &callback) {
void Shard::ReplicationClient::IfStreamingTransaction(const std::function<void(ReplicaStream &handler)> &callback) {
// We can only check the state because it guarantees to be only
// valid during a single transaction replication (if the assumption
// that this and other transaction replication functions can only be
@ -227,7 +220,7 @@ void Storage::ReplicationClient::IfStreamingTransaction(const std::function<void
}
}
void Storage::ReplicationClient::FinalizeTransactionReplication() {
void Shard::ReplicationClient::FinalizeTransactionReplication() {
// We can only check the state because it guarantees to be only
// valid during a single transaction replication (if the assumption
// that this and other transaction replication functions can only be
@ -278,7 +271,7 @@ void Storage::ReplicationClient::FinalizeTransactionReplication() {
}
}
void Storage::ReplicationClient::FinalizeTransactionReplicationInternal() {
void Shard::ReplicationClient::FinalizeTransactionReplicationInternal() {
MG_ASSERT(replica_stream_, "Missing stream for transaction deltas");
try {
auto response = replica_stream_->Finalize();
@ -300,9 +293,9 @@ void Storage::ReplicationClient::FinalizeTransactionReplicationInternal() {
}
}
void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
void Shard::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
while (true) {
auto file_locker = storage_->file_retainer_.AddLocker();
auto file_locker = shard_->file_retainer_.AddLocker();
const auto steps = GetRecoverySteps(replica_commit, &file_locker);
for (const auto &recovery_step : steps) {
@ -319,13 +312,11 @@ void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
auto response = TransferWalFiles(arg);
replica_commit = response.current_commit_timestamp;
} else if constexpr (std::is_same_v<StepType, RecoveryCurrentWal>) {
std::unique_lock transaction_guard(storage_->engine_lock_);
if (storage_->wal_file_ && storage_->wal_file_->SequenceNumber() == arg.current_wal_seq_num) {
storage_->wal_file_->DisableFlushing();
transaction_guard.unlock();
if (shard_->wal_file_ && shard_->wal_file_->SequenceNumber() == arg.current_wal_seq_num) {
shard_->wal_file_->DisableFlushing();
spdlog::debug("Sending current wal file");
replica_commit = ReplicateCurrentWal();
storage_->wal_file_->EnableFlushing();
shard_->wal_file_->EnableFlushing();
}
} else {
static_assert(always_false_v<T>, "Missing type from variant visitor");
@ -354,20 +345,20 @@ void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
// By adding this lock, we can avoid that, and go to RECOVERY immediately.
std::unique_lock client_guard{client_lock_};
SPDLOG_INFO("Replica timestamp: {}", replica_commit);
SPDLOG_INFO("Last commit: {}", storage_->last_commit_timestamp_);
if (storage_->last_commit_timestamp_.load() == replica_commit) {
SPDLOG_INFO("Last commit: {}", shard_->last_commit_timestamp_);
if (shard_->last_commit_timestamp_ == replica_commit) {
replica_state_.store(replication::ReplicaState::READY);
return;
}
}
}
uint64_t Storage::ReplicationClient::ReplicateCurrentWal() {
const auto &wal_file = storage_->wal_file_;
uint64_t Shard::ReplicationClient::ReplicateCurrentWal() {
const auto &wal_file = shard_->wal_file_;
auto stream = TransferCurrentWalFile();
stream.AppendFilename(wal_file->Path().filename());
utils::InputFile file;
MG_ASSERT(file.Open(storage_->wal_file_->Path()), "Failed to open current WAL file!");
MG_ASSERT(file.Open(shard_->wal_file_->Path()), "Failed to open current WAL file!");
const auto [buffer, buffer_size] = wal_file->CurrentFileBuffer();
stream.AppendSize(file.GetSize() + buffer_size);
stream.AppendFileData(&file);
@ -396,23 +387,23 @@ uint64_t Storage::ReplicationClient::ReplicateCurrentWal() {
/// recovery steps, so we can safely send it to the replica.
/// We assume that the property of preserving at least 1 WAL before the snapshot
/// is satisfied as we extract the timestamp information from it.
std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient::GetRecoverySteps(
std::vector<Shard::ReplicationClient::RecoveryStep> Shard::ReplicationClient::GetRecoverySteps(
const uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker) {
// First check if we can recover using the current wal file only
// otherwise save the seq_num of the current wal file
// This lock is also necessary to force the missed transaction to finish.
std::optional<uint64_t> current_wal_seq_num;
std::optional<uint64_t> current_wal_from_timestamp;
if (std::unique_lock transtacion_guard(storage_->engine_lock_); storage_->wal_file_) {
current_wal_seq_num.emplace(storage_->wal_file_->SequenceNumber());
current_wal_from_timestamp.emplace(storage_->wal_file_->FromTimestamp());
if (shard_->wal_file_) {
current_wal_seq_num.emplace(shard_->wal_file_->SequenceNumber());
current_wal_from_timestamp.emplace(shard_->wal_file_->FromTimestamp());
}
auto locker_acc = file_locker->Access();
auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_, current_wal_seq_num);
auto wal_files = durability::GetWalFiles(shard_->wal_directory_, shard_->uuid_, current_wal_seq_num);
MG_ASSERT(wal_files, "Wal files could not be loaded");
auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_);
auto snapshot_files = durability::GetSnapshotFiles(shard_->snapshot_directory_, shard_->uuid_);
std::optional<durability::SnapshotDurabilityInfo> latest_snapshot;
if (!snapshot_files.empty()) {
std::sort(snapshot_files.begin(), snapshot_files.end());
@ -538,13 +529,13 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
}
////// TimeoutDispatcher //////
void Storage::ReplicationClient::TimeoutDispatcher::WaitForTaskToFinish() {
void Shard::ReplicationClient::TimeoutDispatcher::WaitForTaskToFinish() {
// Wait for the previous timeout task to finish
std::unique_lock main_guard(main_lock);
main_cv.wait(main_guard, [&] { return finished; });
}
void Storage::ReplicationClient::TimeoutDispatcher::StartTimeoutTask(const double timeout) {
void Shard::ReplicationClient::TimeoutDispatcher::StartTimeoutTask(const double timeout) {
timeout_pool.AddTask([timeout, this] {
finished = false;
using std::chrono::steady_clock;
@ -562,65 +553,65 @@ void Storage::ReplicationClient::TimeoutDispatcher::StartTimeoutTask(const doubl
});
}
////// ReplicaStream //////
Storage::ReplicationClient::ReplicaStream::ReplicaStream(ReplicationClient *self,
const uint64_t previous_commit_timestamp,
const uint64_t current_seq_num)
Shard::ReplicationClient::ReplicaStream::ReplicaStream(ReplicationClient *self,
const uint64_t previous_commit_timestamp,
const uint64_t current_seq_num)
: self_(self),
stream_(self_->rpc_client_->Stream<replication::AppendDeltasRpc>(previous_commit_timestamp, current_seq_num)) {
replication::Encoder encoder{stream_.GetBuilder()};
encoder.WriteString(self_->storage_->epoch_id_);
encoder.WriteString(self_->shard_->epoch_id_);
}
void Storage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex,
uint64_t final_commit_timestamp) {
void Shard::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex,
uint64_t final_commit_timestamp) {
replication::Encoder encoder(stream_.GetBuilder());
EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, self_->storage_->config_.items, delta, vertex,
EncodeDelta(&encoder, &self_->shard_->name_id_mapper_, self_->shard_->config_.items, delta, vertex,
final_commit_timestamp);
}
void Storage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge,
uint64_t final_commit_timestamp) {
void Shard::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge,
uint64_t final_commit_timestamp) {
replication::Encoder encoder(stream_.GetBuilder());
EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, delta, edge, final_commit_timestamp);
EncodeDelta(&encoder, &self_->shard_->name_id_mapper_, delta, edge, final_commit_timestamp);
}
void Storage::ReplicationClient::ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) {
void Shard::ReplicationClient::ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) {
replication::Encoder encoder(stream_.GetBuilder());
EncodeTransactionEnd(&encoder, final_commit_timestamp);
}
void Storage::ReplicationClient::ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation,
LabelId label, const std::set<PropertyId> &properties,
uint64_t timestamp) {
void Shard::ReplicationClient::ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation,
LabelId label, const std::set<PropertyId> &properties,
uint64_t timestamp) {
replication::Encoder encoder(stream_.GetBuilder());
EncodeOperation(&encoder, &self_->storage_->name_id_mapper_, operation, label, properties, timestamp);
EncodeOperation(&encoder, &self_->shard_->name_id_mapper_, operation, label, properties, timestamp);
}
replication::AppendDeltasRes Storage::ReplicationClient::ReplicaStream::Finalize() { return stream_.AwaitResponse(); }
replication::AppendDeltasRes Shard::ReplicationClient::ReplicaStream::Finalize() { return stream_.AwaitResponse(); }
////// CurrentWalHandler //////
Storage::ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self)
Shard::ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self)
: self_(self), stream_(self_->rpc_client_->Stream<replication::CurrentWalRpc>()) {}
void Storage::ReplicationClient::CurrentWalHandler::AppendFilename(const std::string &filename) {
void Shard::ReplicationClient::CurrentWalHandler::AppendFilename(const std::string &filename) {
replication::Encoder encoder(stream_.GetBuilder());
encoder.WriteString(filename);
}
void Storage::ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) {
void Shard::ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) {
replication::Encoder encoder(stream_.GetBuilder());
encoder.WriteUint(size);
}
void Storage::ReplicationClient::CurrentWalHandler::AppendFileData(utils::InputFile *file) {
void Shard::ReplicationClient::CurrentWalHandler::AppendFileData(utils::InputFile *file) {
replication::Encoder encoder(stream_.GetBuilder());
encoder.WriteFileData(file);
}
void Storage::ReplicationClient::CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) {
void Shard::ReplicationClient::CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) {
replication::Encoder encoder(stream_.GetBuilder());
encoder.WriteBuffer(buffer, buffer_size);
}
replication::CurrentWalRes Storage::ReplicationClient::CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); }
replication::CurrentWalRes Shard::ReplicationClient::CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); }
} // namespace memgraph::storage::v3

8
src/storage/v3/replication/replication_client.hpp
View File

@ -28,7 +28,7 @@
#include "storage/v3/replication/enums.hpp"
#include "storage/v3/replication/rpc.hpp"
#include "storage/v3/replication/serialization.hpp"
#include "storage/v3/storage.hpp"
#include "storage/v3/shard.hpp"
#include "utils/file.hpp"
#include "utils/file_locker.hpp"
#include "utils/spin_lock.hpp"
@ -37,9 +37,9 @@
namespace memgraph::storage::v3 {
class Storage::ReplicationClient {
class Shard::ReplicationClient {
public:
ReplicationClient(std::string name, Storage *storage, const io::network::Endpoint &endpoint,
ReplicationClient(std::string name, Shard *shard, const io::network::Endpoint &endpoint,
replication::ReplicationMode mode, const replication::ReplicationClientConfig &config = {});
// Handler used for transfering the current transaction.
@ -149,7 +149,7 @@ class Storage::ReplicationClient {
void HandleRpcFailure();
std::string name_;
Storage *storage_;
Shard *shard_;
std::optional<communication::ClientContext> rpc_context_;
std::optional<rpc::Client> rpc_client_;

634
src/storage/v3/replication/replication_server.cpp
View File

@ -40,9 +40,9 @@ std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder
};
} // namespace
Storage::ReplicationServer::ReplicationServer(Storage *storage, io::network::Endpoint endpoint,
const replication::ReplicationServerConfig &config)
: storage_(storage) {
Shard::ReplicationServer::ReplicationServer(Shard *shard, io::network::Endpoint endpoint,
const replication::ReplicationServerConfig &config)
: shard_(shard) {
// Create RPC server.
if (config.ssl) {
rpc_server_context_.emplace(config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file,
@ -84,21 +84,21 @@ Storage::ReplicationServer::ReplicationServer(Storage *storage, io::network::End
rpc_server_->Start();
}
void Storage::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::HeartbeatReq req;
slk::Load(&req, req_reader);
replication::HeartbeatRes res{true, storage_->last_commit_timestamp_.load(), storage_->epoch_id_};
replication::HeartbeatRes res{true, shard_->last_commit_timestamp_, shard_->epoch_id_};
slk::Save(res, res_builder);
}
void Storage::ReplicationServer::FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::FrequentHeartbeatReq req;
slk::Load(&req, req_reader);
replication::FrequentHeartbeatRes res{true};
slk::Save(res, res_builder);
}
void Storage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::AppendDeltasReq req;
slk::Load(&req, req_reader);
@ -107,25 +107,25 @@ void Storage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, sl
auto maybe_epoch_id = decoder.ReadString();
MG_ASSERT(maybe_epoch_id, "Invalid replication message");
if (*maybe_epoch_id != storage_->epoch_id_) {
storage_->epoch_history_.emplace_back(std::move(storage_->epoch_id_), storage_->last_commit_timestamp_);
storage_->epoch_id_ = std::move(*maybe_epoch_id);
if (*maybe_epoch_id != shard_->epoch_id_) {
shard_->epoch_history_.emplace_back(std::move(shard_->epoch_id_), shard_->last_commit_timestamp_);
shard_->epoch_id_ = std::move(*maybe_epoch_id);
}
if (storage_->wal_file_) {
if (req.seq_num > storage_->wal_file_->SequenceNumber() || *maybe_epoch_id != storage_->epoch_id_) {
storage_->wal_file_->FinalizeWal();
storage_->wal_file_.reset();
storage_->wal_seq_num_ = req.seq_num;
if (shard_->wal_file_) {
if (req.seq_num > shard_->wal_file_->SequenceNumber() || *maybe_epoch_id != shard_->epoch_id_) {
shard_->wal_file_->FinalizeWal();
shard_->wal_file_.reset();
shard_->wal_seq_num_ = req.seq_num;
} else {
MG_ASSERT(storage_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
storage_->wal_seq_num_ = req.seq_num + 1;
MG_ASSERT(shard_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
shard_->wal_seq_num_ = req.seq_num + 1;
}
} else {
storage_->wal_seq_num_ = req.seq_num;
shard_->wal_seq_num_ = req.seq_num;
}
if (req.previous_commit_timestamp != storage_->last_commit_timestamp_.load()) {
if (req.previous_commit_timestamp != shard_->last_commit_timestamp_) {
// Empty the stream
bool transaction_complete = false;
while (!transaction_complete) {
@ -134,83 +134,81 @@ void Storage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, sl
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
}
replication::AppendDeltasRes res{false, storage_->last_commit_timestamp_.load()};
replication::AppendDeltasRes res{false, shard_->last_commit_timestamp_};
slk::Save(res, res_builder);
return;
}
ReadAndApplyDelta(&decoder);
replication::AppendDeltasRes res{true, storage_->last_commit_timestamp_.load()};
replication::AppendDeltasRes res{true, shard_->last_commit_timestamp_};
slk::Save(res, res_builder);
}
void Storage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::SnapshotReq req;
slk::Load(&req, req_reader);
replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->snapshot_directory_);
utils::EnsureDirOrDie(shard_->snapshot_directory_);
const auto maybe_snapshot_path = decoder.ReadFile(storage_->snapshot_directory_);
const auto maybe_snapshot_path = decoder.ReadFile(shard_->snapshot_directory_);
MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!");
spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path);
std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
// Clear the database
storage_->vertices_.clear();
storage_->edges_.clear();
shard_->vertices_.clear();
shard_->edges_.clear();
storage_->constraints_ = Constraints();
storage_->indices_.label_index =
LabelIndex(&storage_->indices_, &storage_->constraints_, storage_->config_.items, storage_->schema_validator_);
storage_->indices_.label_property_index = LabelPropertyIndex(&storage_->indices_, &storage_->constraints_,
storage_->config_.items, storage_->schema_validator_);
shard_->constraints_ = Constraints();
shard_->indices_.label_index =
LabelIndex(&shard_->indices_, &shard_->constraints_, shard_->config_.items, shard_->vertex_validator_);
shard_->indices_.label_property_index =
LabelPropertyIndex(&shard_->indices_, &shard_->constraints_, shard_->config_.items, shard_->vertex_validator_);
try {
spdlog::debug("Loading snapshot");
auto recovered_snapshot = durability::LoadSnapshot(*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_,
&storage_->epoch_history_, &storage_->name_id_mapper_,
&storage_->edge_count_, storage_->config_.items);
auto recovered_snapshot = durability::RecoveredSnapshot{};
// durability::LoadSnapshot(*maybe_snapshot_path, &shard_->vertices_, &shard_->edges_,
// &shard_->epoch_history_,
// &shard_->name_id_mapper_, &shard_->edge_count_, shard_->config_.items);
spdlog::debug("Snapshot loaded successfully");
// If this step is present it should always be the first step of
// the recovery so we use the UUID we read from snasphost
storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
storage_->epoch_id_ = std::move(recovered_snapshot.snapshot_info.epoch_id);
shard_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
shard_->epoch_id_ = std::move(recovered_snapshot.snapshot_info.epoch_id);
const auto &recovery_info = recovered_snapshot.recovery_info;
storage_->vertex_id_ = recovery_info.next_vertex_id;
storage_->edge_id_ = recovery_info.next_edge_id;
storage_->timestamp_ = std::max(storage_->timestamp_, recovery_info.next_timestamp);
shard_->timestamp_ = std::max(shard_->timestamp_, recovery_info.next_timestamp);
durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage_->indices_,
&storage_->constraints_, &storage_->vertices_);
// durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &shard_->indices_,
// &shard_->constraints_, &shard_->vertices_);
} catch (const durability::RecoveryFailure &e) {
LOG_FATAL("Couldn't load the snapshot because of: {}", e.what());
}
storage_guard.unlock();
replication::SnapshotRes res{true, storage_->last_commit_timestamp_.load()};
replication::SnapshotRes res{true, shard_->last_commit_timestamp_};
slk::Save(res, res_builder);
// Delete other durability files
auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_);
auto snapshot_files = durability::GetSnapshotFiles(shard_->snapshot_directory_, shard_->uuid_);
for (const auto &[path, uuid, _] : snapshot_files) {
if (path != *maybe_snapshot_path) {
storage_->file_retainer_.DeleteFile(path);
shard_->file_retainer_.DeleteFile(path);
}
}
auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_);
auto wal_files = durability::GetWalFiles(shard_->wal_directory_, shard_->uuid_);
if (wal_files) {
for (const auto &wal_file : *wal_files) {
storage_->file_retainer_.DeleteFile(wal_file.path);
shard_->file_retainer_.DeleteFile(wal_file.path);
}
storage_->wal_file_.reset();
shard_->wal_file_.reset();
}
}
void Storage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::WalFilesReq req;
slk::Load(&req, req_reader);
@ -219,31 +217,31 @@ void Storage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::B
replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_);
utils::EnsureDirOrDie(shard_->wal_directory_);
for (auto i = 0; i < wal_file_number; ++i) {
LoadWal(&decoder);
}
replication::WalFilesRes res{true, storage_->last_commit_timestamp_.load()};
replication::WalFilesRes res{true, shard_->last_commit_timestamp_};
slk::Save(res, res_builder);
}
void Storage::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
void Shard::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::CurrentWalReq req;
slk::Load(&req, req_reader);
replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_);
utils::EnsureDirOrDie(shard_->wal_directory_);
LoadWal(&decoder);
replication::CurrentWalRes res{true, storage_->last_commit_timestamp_.load()};
replication::CurrentWalRes res{true, shard_->last_commit_timestamp_};
slk::Save(res, res_builder);
}
void Storage::ReplicationServer::LoadWal(replication::Decoder *decoder) {
void Shard::ReplicationServer::LoadWal(replication::Decoder *decoder) {
const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory;
utils::EnsureDir(temp_wal_directory);
auto maybe_wal_path = decoder->ReadFile(temp_wal_directory);
@ -252,22 +250,22 @@ void Storage::ReplicationServer::LoadWal(replication::Decoder *decoder) {
try {
auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
if (wal_info.seq_num == 0) {
storage_->uuid_ = wal_info.uuid;
shard_->uuid_ = wal_info.uuid;
}
if (wal_info.epoch_id != storage_->epoch_id_) {
storage_->epoch_history_.emplace_back(wal_info.epoch_id, storage_->last_commit_timestamp_);
storage_->epoch_id_ = std::move(wal_info.epoch_id);
if (wal_info.epoch_id != shard_->epoch_id_) {
shard_->epoch_history_.emplace_back(wal_info.epoch_id, shard_->last_commit_timestamp_);
shard_->epoch_id_ = std::move(wal_info.epoch_id);
}
if (storage_->wal_file_) {
if (storage_->wal_file_->SequenceNumber() != wal_info.seq_num) {
storage_->wal_file_->FinalizeWal();
storage_->wal_seq_num_ = wal_info.seq_num;
storage_->wal_file_.reset();
if (shard_->wal_file_) {
if (shard_->wal_file_->SequenceNumber() != wal_info.seq_num) {
shard_->wal_file_->FinalizeWal();
shard_->wal_seq_num_ = wal_info.seq_num;
shard_->wal_file_.reset();
}
} else {
storage_->wal_seq_num_ = wal_info.seq_num;
shard_->wal_seq_num_ = wal_info.seq_num;
}
durability::Decoder wal;
@ -286,28 +284,28 @@ void Storage::ReplicationServer::LoadWal(replication::Decoder *decoder) {
}
}
Storage::ReplicationServer::~ReplicationServer() {
Shard::ReplicationServer::~ReplicationServer() {
if (rpc_server_) {
rpc_server_->Shutdown();
rpc_server_->AwaitShutdown();
}
}
uint64_t Storage::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *decoder) {
auto edge_acc = storage_->edges_.access();
auto vertex_acc = storage_->vertices_.access();
uint64_t Shard::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *decoder) {
auto edge_acc = shard_->edges_.access();
// auto vertex_acc = shard_->vertices_.access();
std::optional<std::pair<uint64_t, Storage::Accessor>> commit_timestamp_and_accessor;
auto get_transaction = [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
if (!commit_timestamp_and_accessor) {
commit_timestamp_and_accessor.emplace(commit_timestamp, storage_->Access());
} else if (commit_timestamp_and_accessor->first != commit_timestamp) {
throw utils::BasicException("Received more than one transaction!");
}
return &commit_timestamp_and_accessor->second;
};
std::optional<std::pair<uint64_t, Shard::Accessor>> commit_timestamp_and_accessor;
// auto get_transaction = [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
// if (!commit_timestamp_and_accessor) {
// commit_timestamp_and_accessor.emplace(commit_timestamp, shard_->Access());
// } else if (commit_timestamp_and_accessor->first != commit_timestamp) {
// throw utils::BasicException("Received more than one transaction!");
// }
// return &commit_timestamp_and_accessor->second;
// };
uint64_t applied_deltas = 0;
auto max_commit_timestamp = storage_->last_commit_timestamp_.load();
auto max_commit_timestamp = shard_->last_commit_timestamp_;
for (bool transaction_complete = false; !transaction_complete; ++applied_deltas) {
const auto [timestamp, delta] = ReadDelta(decoder);
@ -317,259 +315,255 @@ uint64_t Storage::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
if (timestamp < storage_->timestamp_) {
if (timestamp < shard_->timestamp_) {
continue;
}
SPDLOG_INFO(" Delta {}", applied_deltas);
switch (delta.type) {
case durability::WalDeltaData::Type::VERTEX_CREATE: {
spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp);
transaction->CreateVertex(delta.vertex_create_delete.gid);
break;
}
case durability::WalDeltaData::Type::VERTEX_DELETE: {
spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW);
if (!vertex) throw utils::BasicException("Invalid transaction!");
auto ret = transaction->DeleteVertex(&*vertex);
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
if (!vertex) throw utils::BasicException("Invalid transaction!");
auto ret = vertex->AddLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
if (!vertex) throw utils::BasicException("Invalid transaction!");
auto ret = vertex->RemoveLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_edge_set_property.gid, View::NEW);
if (!vertex) throw utils::BasicException("Invalid transaction!");
auto ret = vertex->SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
delta.vertex_edge_set_property.value);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_CREATE: {
spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
auto *transaction = get_transaction(timestamp);
auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW);
if (!from_vertex) throw utils::BasicException("Invalid transaction!");
auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW);
if (!to_vertex) throw utils::BasicException("Invalid transaction!");
auto edge = transaction->CreateEdge(&*from_vertex, &*to_vertex,
transaction->NameToEdgeType(delta.edge_create_delete.edge_type),
delta.edge_create_delete.gid);
if (edge.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_DELETE: {
spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
auto *transaction = get_transaction(timestamp);
auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW);
if (!from_vertex) throw utils::BasicException("Invalid transaction!");
auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW);
if (!to_vertex) throw utils::BasicException("Invalid transaction!");
auto edges = from_vertex->OutEdges(View::NEW, {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)},
&*to_vertex);
if (edges.HasError()) throw utils::BasicException("Invalid transaction!");
if (edges->size() != 1) throw utils::BasicException("Invalid transaction!");
auto &edge = (*edges)[0];
auto ret = transaction->DeleteEdge(&edge);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
// SPDLOG_INFO(" Delta {}", applied_deltas);
// switch (delta.type) {
// case durability::WalDeltaData::Type::VERTEX_CREATE: {
// spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint());
// auto *transaction = get_transaction(timestamp);
// transaction->CreateVertex(delta.vertex_create_delete.gid);
// break;
// }
// case durability::WalDeltaData::Type::VERTEX_DELETE: {
// spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint());
// auto *transaction = get_transaction(timestamp);
// auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW);
// if (!vertex) throw utils::BasicException("Invalid transaction!");
// auto ret = transaction->DeleteVertex(&*vertex);
// if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
// spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(),
// delta.vertex_add_remove_label.label);
// auto *transaction = get_transaction(timestamp);
// auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
// if (!vertex) throw utils::BasicException("Invalid transaction!");
// auto ret = vertex->AddLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
// if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
// spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(),
// delta.vertex_add_remove_label.label);
// auto *transaction = get_transaction(timestamp);
// auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
// if (!vertex) throw utils::BasicException("Invalid transaction!");
// auto ret = vertex->RemoveLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
// if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
// spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
// delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
// auto *transaction = get_transaction(timestamp);
// auto vertex = transaction->FindVertex(delta.vertex_edge_set_property.gid, View::NEW);
// if (!vertex) throw utils::BasicException("Invalid transaction!");
// auto ret = vertex->SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
// delta.vertex_edge_set_property.value);
// if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::EDGE_CREATE: {
// spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}",
// delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
// delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
// auto *transaction = get_transaction(timestamp);
// auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW);
// if (!from_vertex) throw utils::BasicException("Invalid transaction!");
// auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW);
// if (!to_vertex) throw utils::BasicException("Invalid transaction!");
// auto edge = transaction->CreateEdge(&*from_vertex, &*to_vertex,
// transaction->NameToEdgeType(delta.edge_create_delete.edge_type),
// delta.edge_create_delete.gid);
// if (edge.HasError()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::EDGE_DELETE: {
// spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}",
// delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
// delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
// auto *transaction = get_transaction(timestamp);
// auto from_vertex = transaction->FindVertex(delta.edge_create_delete.from_vertex, View::NEW);
// if (!from_vertex) throw utils::BasicException("Invalid transaction!");
// auto to_vertex = transaction->FindVertex(delta.edge_create_delete.to_vertex, View::NEW);
// if (!to_vertex) throw utils::BasicException("Invalid transaction!");
// auto edges = from_vertex->OutEdges(View::NEW,
// {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)},
// &*to_vertex);
// if (edges.HasError()) throw utils::BasicException("Invalid transaction!");
// if (edges->size() != 1) throw utils::BasicException("Invalid transaction!");
// auto &edge = (*edges)[0];
// auto ret = transaction->DeleteEdge(&edge);
// if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
// spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
// delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
if (!storage_->config_.items.properties_on_edges)
throw utils::BasicException(
"Can't set properties on edges because properties on edges "
"are disabled!");
// if (!shard_->config_.items.properties_on_edges)
// throw utils::BasicException(
// "Can't set properties on edges because properties on edges "
// "are disabled!");
auto *transaction = get_transaction(timestamp);
// // auto *transaction = get_transaction(timestamp);
// The following block of code effectively implements `FindEdge` and
// yields an accessor that is only valid for managing the edge's
// properties.
auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
if (edge == edge_acc.end()) throw utils::BasicException("Invalid transaction!");
// The edge visibility check must be done here manually because we
// don't allow direct access to the edges through the public API.
{
bool is_visible = true;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge->lock);
is_visible = !edge->deleted;
delta = edge->delta;
}
ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW, [&is_visible](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL:
case Delta::Action::SET_PROPERTY:
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::REMOVE_OUT_EDGE:
break;
case Delta::Action::RECREATE_OBJECT: {
is_visible = true;
break;
}
case Delta::Action::DELETE_OBJECT: {
is_visible = false;
break;
}
}
});
if (!is_visible) throw utils::BasicException("Invalid transaction!");
}
EdgeRef edge_ref(&*edge);
// Here we create an edge accessor that we will use to get the
// properties of the edge. The accessor is created with an invalid
// type and invalid from/to pointers because we don't know them
// here, but that isn't an issue because we won't use that part of
// the API here.
auto ea = EdgeAccessor{edge_ref,
EdgeTypeId::FromUint(0UL),
nullptr,
nullptr,
&transaction->transaction_,
&storage_->indices_,
&storage_->constraints_,
storage_->config_.items,
storage_->schema_validator_};
// // The following block of code effectively implements `FindEdge` and
// // yields an accessor that is only valid for managing the edge's
// // properties.
// auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
// if (edge == edge_acc.end()) throw utils::BasicException("Invalid transaction!");
// // The edge visibility check must be done here manually because we
// // don't allow direct access to the edges through the public API.
// {
// auto is_visible = !edge->deleted;
// auto *delta = edge->delta;
// ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW, [&is_visible](const Delta &delta) {
// switch (delta.action) {
// case Delta::Action::ADD_LABEL:
// case Delta::Action::REMOVE_LABEL:
// case Delta::Action::SET_PROPERTY:
// case Delta::Action::ADD_IN_EDGE:
// case Delta::Action::ADD_OUT_EDGE:
// case Delta::Action::REMOVE_IN_EDGE:
// case Delta::Action::REMOVE_OUT_EDGE:
// break;
// case Delta::Action::RECREATE_OBJECT: {
// is_visible = true;
// break;
// }
// case Delta::Action::DELETE_OBJECT: {
// is_visible = false;
// break;
// }
// }
// });
// if (!is_visible) throw utils::BasicException("Invalid transaction!");
// }
// EdgeRef edge_ref(&*edge);
// // Here we create an edge accessor that we will use to get the
// // properties of the edge. The accessor is created with an invalid
// // type and invalid from/to pointers because we don't know them
// // here, but that isn't an issue because we won't use that part of
// // the API here.
// auto ea = EdgeAccessor{edge_ref,
// EdgeTypeId::FromUint(0UL),
// nullptr,
// nullptr,
// &transaction->transaction_,
// &shard_->indices_,
// &shard_->constraints_,
// shard_->config_.items,
// shard_->schema_validator_};
auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
delta.vertex_edge_set_property.value);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
// auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
// delta.vertex_edge_set_property.value);
// if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
// break;
// }
case durability::WalDeltaData::Type::TRANSACTION_END: {
spdlog::trace(" Transaction end");
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
throw utils::BasicException("Invalid data!");
auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
commit_timestamp_and_accessor = std::nullopt;
break;
}
// case durability::WalDeltaData::Type::TRANSACTION_END: {
// spdlog::trace(" Transaction end");
// if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
// throw utils::BasicException("Invalid data!");
// auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first);
// if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
// commit_timestamp_and_accessor = std::nullopt;
// break;
// }
case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
spdlog::trace(" Create label index on :{}", delta.operation_label.label);
// Need to send the timestamp
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
if (!storage_->CreateIndex(storage_->NameToLabel(delta.operation_label.label), timestamp))
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
spdlog::trace(" Drop label index on :{}", delta.operation_label.label);
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
if (!storage_->DropIndex(storage_->NameToLabel(delta.operation_label.label), timestamp))
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
if (!storage_->CreateIndex(storage_->NameToLabel(delta.operation_label_property.label),
storage_->NameToProperty(delta.operation_label_property.property), timestamp))
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
if (!storage_->DropIndex(storage_->NameToLabel(delta.operation_label_property.label),
storage_->NameToProperty(delta.operation_label_property.property), timestamp))
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
auto ret = storage_->CreateExistenceConstraint(
storage_->NameToLabel(delta.operation_label_property.label),
storage_->NameToProperty(delta.operation_label_property.property), timestamp);
if (!ret.HasValue() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
if (!storage_->DropExistenceConstraint(storage_->NameToLabel(delta.operation_label_property.label),
storage_->NameToProperty(delta.operation_label_property.property),
timestamp))
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
std::set<PropertyId> properties;
for (const auto &prop : delta.operation_label_properties.properties) {
properties.emplace(storage_->NameToProperty(prop));
}
auto ret = storage_->CreateUniqueConstraint(storage_->NameToLabel(delta.operation_label_properties.label),
properties, timestamp);
if (!ret.HasValue() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
std::set<PropertyId> properties;
for (const auto &prop : delta.operation_label_properties.properties) {
properties.emplace(storage_->NameToProperty(prop));
}
auto ret = storage_->DropUniqueConstraint(storage_->NameToLabel(delta.operation_label_properties.label),
properties, timestamp);
if (ret != UniqueConstraints::DeletionStatus::SUCCESS) throw utils::BasicException("Invalid transaction!");
break;
}
}
// case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
// spdlog::trace(" Create label index on :{}", delta.operation_label.label);
// // Need to send the timestamp
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// if (!shard_->CreateIndex(shard_->NameToLabel(delta.operation_label.label), timestamp))
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
// spdlog::trace(" Drop label index on :{}", delta.operation_label.label);
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// if (!shard_->DropIndex(shard_->NameToLabel(delta.operation_label.label), timestamp))
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
// spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label,
// delta.operation_label_property.property);
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// if (!shard_->CreateIndex(shard_->NameToLabel(delta.operation_label_property.label),
// shard_->NameToProperty(delta.operation_label_property.property), timestamp))
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
// spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label,
// delta.operation_label_property.property);
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// if (!shard_->DropIndex(shard_->NameToLabel(delta.operation_label_property.label),
// shard_->NameToProperty(delta.operation_label_property.property), timestamp))
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
// spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
// delta.operation_label_property.property);
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// auto ret = shard_->CreateExistenceConstraint(
// shard_->NameToLabel(delta.operation_label_property.label),
// shard_->NameToProperty(delta.operation_label_property.property), timestamp);
// if (!ret.HasValue() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
// spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label,
// delta.operation_label_property.property);
// if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// if (!shard_->DropExistenceConstraint(shard_->NameToLabel(delta.operation_label_property.label),
// shard_->NameToProperty(delta.operation_label_property.property),
// timestamp))
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
// std::stringstream ss;
// utils::PrintIterable(ss, delta.operation_label_properties.properties);
// spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label,
// ss.str()); if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// std::set<PropertyId> properties;
// for (const auto &prop : delta.operation_label_properties.properties) {
// properties.emplace(shard_->NameToProperty(prop));
// }
// auto ret = shard_->CreateUniqueConstraint(shard_->NameToLabel(delta.operation_label_properties.label),
// properties, timestamp);
// if (!ret.HasValue() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
// throw utils::BasicException("Invalid transaction!");
// break;
// }
// case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
// std::stringstream ss;
// utils::PrintIterable(ss, delta.operation_label_properties.properties);
// spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label,
// ss.str()); if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!");
// std::set<PropertyId> properties;
// for (const auto &prop : delta.operation_label_properties.properties) {
// properties.emplace(shard_->NameToProperty(prop));
// }
// auto ret = shard_->DropUniqueConstraint(shard_->NameToLabel(delta.operation_label_properties.label),
// properties, timestamp);
// if (ret != UniqueConstraints::DeletionStatus::SUCCESS) throw utils::BasicException("Invalid transaction!");
// break;
// }
// }
}
if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid data!");
storage_->last_commit_timestamp_ = max_commit_timestamp;
shard_->last_commit_timestamp_ = max_commit_timestamp;
return applied_deltas;
}

8
src/storage/v3/replication/replication_server.hpp
View File

@ -11,13 +11,13 @@
#pragma once
#include "storage/v3/storage.hpp"
#include "storage/v3/shard.hpp"
namespace memgraph::storage::v3 {
class Storage::ReplicationServer {
class Shard::ReplicationServer {
public:
explicit ReplicationServer(Storage *storage, io::network::Endpoint endpoint,
explicit ReplicationServer(Shard *shard, io::network::Endpoint endpoint,
const replication::ReplicationServerConfig &config);
ReplicationServer(const ReplicationServer &) = delete;
ReplicationServer(ReplicationServer &&) = delete;
@ -41,7 +41,7 @@ class Storage::ReplicationServer {
std::optional<communication::ServerContext> rpc_server_context_;
std::optional<rpc::Server> rpc_server_;
Storage *storage_;
Shard *shard_;
};
} // namespace memgraph::storage::v3

17
src/storage/v3/schema_validator.cpp
View File

@ -98,9 +98,24 @@ SchemaValidator::SchemaValidator(Schemas &schemas) : schemas_{schemas} {}
[[nodiscard]] std::optional<SchemaViolation> SchemaValidator::ValidateLabelUpdate(const LabelId label) const {
const auto *schema = schemas_.GetSchema(label);
if (schema) {
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_MODIFY_PRIMARY_LABEL, label);
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label);
}
return std::nullopt;
}
VertexValidator::VertexValidator(const SchemaValidator &schema_validator, const LabelId primary_label)
: schema_validator{&schema_validator}, primary_label_{primary_label} {}
[[nodiscard]] std::optional<SchemaViolation> VertexValidator::ValidatePropertyUpdate(PropertyId property_id) const {
return schema_validator->ValidatePropertyUpdate(primary_label_, property_id);
};
[[nodiscard]] std::optional<SchemaViolation> VertexValidator::ValidateAddLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}
[[nodiscard]] std::optional<SchemaViolation> VertexValidator::ValidateRemoveLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}
} // namespace memgraph::storage::v3

16
src/storage/v3/schema_validator.hpp
View File

@ -27,7 +27,7 @@ struct SchemaViolation {
NO_SCHEMA_DEFINED_FOR_LABEL,
VERTEX_PROPERTY_WRONG_TYPE,
VERTEX_UPDATE_PRIMARY_KEY,
VERTEX_MODIFY_PRIMARY_LABEL,
VERTEX_UPDATE_PRIMARY_LABEL,
VERTEX_SECONDARY_LABEL_IS_PRIMARY,
};
@ -63,6 +63,20 @@ class SchemaValidator {
Schemas &schemas_;
};
struct VertexValidator {
explicit VertexValidator(const SchemaValidator &schema_validator, LabelId primary_label);
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(PropertyId property_id) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateAddLabel(LabelId label) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateRemoveLabel(LabelId label) const;
const SchemaValidator *schema_validator;
LabelId primary_label_;
};
template <typename TValue>
using ResultSchema = utils::BasicResult<std::variant<SchemaViolation, Error>, TValue>;

11
src/storage/v3/schemas.cpp
View File

@ -11,10 +11,12 @@
#include "storage/v3/schemas.hpp"
#include <algorithm>
#include <unordered_map>
#include <vector>
#include "storage/v3/property_value.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::storage::v3 {
@ -47,6 +49,15 @@ bool Schemas::CreateSchema(const LabelId primary_label, const std::vector<Schema
bool Schemas::DropSchema(const LabelId primary_label) { return schemas_.erase(primary_label); }
bool Schemas::IsPropertyKey(const LabelId primary_label, const PropertyId property_id) const {
if (const auto schema = schemas_.find(primary_label); schema != schemas_.end()) {
return std::ranges::find_if(schema->second, [property_id](const auto &elem) {
return elem.property_id == property_id;
}) != schema->second.end();
}
throw utils::BasicException("Schema not found!");
}
std::optional<common::SchemaType> PropertyTypeToSchemaType(const PropertyValue &property_value) {
switch (property_value.type()) {
case PropertyValue::Type::Bool: {

8
src/storage/v3/schemas.hpp
View File

@ -53,11 +53,15 @@ class Schemas {
// Returns true if it was successfully created or false if the schema
// already exists
[[nodiscard]] bool CreateSchema(LabelId label, const std::vector<SchemaProperty> &schemas_types);
[[nodiscard]] bool CreateSchema(LabelId primary_label, const std::vector<SchemaProperty> &schemas_types);
// Returns true if it was successfully dropped or false if the schema
// does not exist
[[nodiscard]] bool DropSchema(LabelId label);
[[nodiscard]] bool DropSchema(LabelId primary_label);
// Returns true if property is part of schema defined
// by primary label
[[nodiscard]] bool IsPropertyKey(LabelId primary_label, PropertyId property_id) const;
private:
SchemasMap schemas_;

1900
src/storage/v3/shard.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

626
src/storage/v3/shard.hpp Normal file
View File

@ -0,0 +1,626 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <cstdint>
#include <filesystem>
#include <map>
#include <numeric>
#include <optional>
#include <shared_mutex>
#include <variant>
#include <vector>
#include "io/network/endpoint.hpp"
#include "kvstore/kvstore.hpp"
#include "storage/v3/commit_log.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/constraints.hpp"
#include "storage/v3/durability/metadata.hpp"
#include "storage/v3/durability/wal.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertex_id.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/exceptions.hpp"
#include "utils/file_locker.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/rw_lock.hpp"
#include "utils/scheduler.hpp"
#include "utils/skip_list.hpp"
#include "utils/synchronized.hpp"
#include "utils/uuid.hpp"
/// REPLICATION ///
#include "rpc/server.hpp"
#include "storage/v3/replication/config.hpp"
#include "storage/v3/replication/enums.hpp"
#include "storage/v3/replication/rpc.hpp"
#include "storage/v3/replication/serialization.hpp"
namespace memgraph::storage::v3 {
// The storage is based on this paper:
// https://db.in.tum.de/~muehlbau/papers/mvcc.pdf
// The paper implements a fully serializable storage, in our implementation we
// only implement snapshot isolation for transactions.
/// Iterable for iterating through all vertices of a Storage.
///
/// An instance of this will be usually be wrapped inside VerticesIterable for
/// generic, public use.
class AllVerticesIterable final {
VerticesSkipList::Accessor vertices_accessor_;
Transaction *transaction_;
View view_;
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const VertexValidator *vertex_validator_;
const Schemas *schemas_;
std::optional<VertexAccessor> vertex_;
public:
class Iterator final {
AllVerticesIterable *self_;
VerticesSkipList::Iterator it_;
public:
Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it);
VertexAccessor operator*() const;
Iterator &operator++();
bool operator==(const Iterator &other) const { return self_ == other.self_ && it_ == other.it_; }
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view,
Indices *indices, Constraints *constraints, Config::Items config,
const VertexValidator &vertex_validator)
: vertices_accessor_(std::move(vertices_accessor)),
transaction_(transaction),
view_(view),
indices_(indices),
constraints_(constraints),
config_(config),
vertex_validator_{&vertex_validator} {}
Iterator begin() { return {this, vertices_accessor_.begin()}; }
Iterator end() { return {this, vertices_accessor_.end()}; }
};
/// Generic access to different kinds of vertex iterations.
///
/// This class should be the primary type used by the client code to iterate
/// over vertices inside a Storage instance.
class VerticesIterable final {
enum class Type { ALL, BY_LABEL, BY_LABEL_PROPERTY };
Type type_;
union {
AllVerticesIterable all_vertices_;
LabelIndex::Iterable vertices_by_label_;
LabelPropertyIndex::Iterable vertices_by_label_property_;
};
public:
explicit VerticesIterable(AllVerticesIterable);
explicit VerticesIterable(LabelIndex::Iterable);
explicit VerticesIterable(LabelPropertyIndex::Iterable);
VerticesIterable(const VerticesIterable &) = delete;
VerticesIterable &operator=(const VerticesIterable &) = delete;
VerticesIterable(VerticesIterable &&) noexcept;
VerticesIterable &operator=(VerticesIterable &&) noexcept;
~VerticesIterable();
class Iterator final {
Type type_;
union {
AllVerticesIterable::Iterator all_it_;
LabelIndex::Iterable::Iterator by_label_it_;
LabelPropertyIndex::Iterable::Iterator by_label_property_it_;
};
void Destroy() noexcept;
public:
explicit Iterator(AllVerticesIterable::Iterator);
explicit Iterator(LabelIndex::Iterable::Iterator);
explicit Iterator(LabelPropertyIndex::Iterable::Iterator);
Iterator(const Iterator &);
Iterator &operator=(const Iterator &);
Iterator(Iterator &&) noexcept;
Iterator &operator=(Iterator &&) noexcept;
~Iterator();
VertexAccessor operator*() const;
Iterator &operator++();
bool operator==(const Iterator &other) const;
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
Iterator begin();
Iterator end();
};
/// Structure used to return information about existing indices in the storage.
struct IndicesInfo {
std::vector<LabelId> label;
std::vector<std::pair<LabelId, PropertyId>> label_property;
};
/// Structure used to return information about existing constraints in the
/// storage.
struct ConstraintsInfo {
std::vector<std::pair<LabelId, PropertyId>> existence;
std::vector<std::pair<LabelId, std::set<PropertyId>>> unique;
};
/// Structure used to return information about existing schemas in the storage
struct SchemasInfo {
Schemas::SchemasList schemas;
};
/// Structure used to return information about the storage.
struct StorageInfo {
uint64_t vertex_count;
uint64_t edge_count;
double average_degree;
uint64_t memory_usage;
uint64_t disk_usage;
};
enum class ReplicationRole : uint8_t { MAIN, REPLICA };
class Shard final {
public:
/// @throw std::system_error
/// @throw std::bad_alloc
explicit Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
Config config = Config());
Shard(const Shard &) = delete;
Shard(Shard &&) noexcept = delete;
Shard &operator=(const Shard &) = delete;
Shard operator=(Shard &&) noexcept = delete;
~Shard();
class Accessor final {
private:
friend class Shard;
explicit Accessor(Shard *shard, IsolationLevel isolation_level);
public:
Accessor(const Accessor &) = delete;
Accessor &operator=(const Accessor &) = delete;
Accessor &operator=(Accessor &&other) = delete;
// NOTE: After the accessor is moved, all objects derived from it (accessors
// and iterators) are *invalid*. You have to get all derived objects again.
Accessor(Accessor &&other) noexcept;
~Accessor();
// TODO(gvolfing) this is just a workaround for stitching remove this later.
LabelId GetPrimaryLabel() const noexcept { return shard_->primary_label_; }
/// @throw std::bad_alloc
ResultSchema<VertexAccessor> CreateVertexAndValidate(
LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
/// @throw std::bad_alloc
ResultSchema<VertexAccessor> CreateVertexAndValidate(
LabelId primary_label, const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
std::optional<VertexAccessor> FindVertex(std::vector<PropertyValue> primary_key, View view);
VerticesIterable Vertices(View view) {
return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), &transaction_, view, &shard_->indices_,
&shard_->constraints_, shard_->config_.items,
shard_->vertex_validator_));
}
VerticesIterable Vertices(LabelId label, View view);
VerticesIterable Vertices(LabelId label, PropertyId property, View view);
VerticesIterable Vertices(LabelId label, PropertyId property, const PropertyValue &value, View view);
VerticesIterable Vertices(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view);
/// Return approximate number of all vertices in the database.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount() const { return static_cast<int64_t>(shard_->vertices_.size()); }
/// Return approximate number of vertices with the given label.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label) const {
return shard_->indices_.label_index.ApproximateVertexCount(label);
}
/// Return approximate number of vertices with the given label and property.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property);
}
/// Return approximate number of vertices with the given label and the given
/// value for the given property. Note that this is always an over-estimate
/// and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
}
/// Return approximate number of vertices with the given label and value for
/// the given property in the range defined by provided upper and lower
/// bounds.
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const {
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
}
/// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
/// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
VertexAccessor *vertex);
/// @throw std::bad_alloc
Result<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<EdgeAccessor>> DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, Gid edge_id);
const std::string &LabelToName(LabelId label) const;
const std::string &PropertyToName(PropertyId property) const;
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const;
bool LabelIndexExists(LabelId label) const { return shard_->indices_.label_index.IndexExists(label); }
bool LabelPropertyIndexExists(LabelId label, PropertyId property) const {
return shard_->indices_.label_property_index.IndexExists(label, property);
}
IndicesInfo ListAllIndices() const {
return {shard_->indices_.label_index.ListIndices(), shard_->indices_.label_property_index.ListIndices()};
}
ConstraintsInfo ListAllConstraints() const {
return {ListExistenceConstraints(shard_->constraints_),
shard_->constraints_.unique_constraints.ListConstraints()};
}
const SchemaValidator &GetSchemaValidator() const;
SchemasInfo ListAllSchemas() const { return {shard_->schemas_.ListSchemas()}; }
void AdvanceCommand();
/// Commit returns `ConstraintViolation` if the changes made by this
/// transaction violate an existence or unique constraint. In that case the
/// transaction is automatically aborted. Otherwise, void is returned.
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, void> Commit(std::optional<uint64_t> desired_commit_timestamp = {});
/// @throw std::bad_alloc
void Abort();
void FinalizeTransaction();
private:
/// @throw std::bad_alloc
VertexAccessor CreateVertex(Gid gid, LabelId primary_label);
Shard *shard_;
Transaction transaction_;
std::optional<uint64_t> commit_timestamp_;
bool is_transaction_active_;
Config::Items config_;
};
Accessor Access(std::optional<IsolationLevel> override_isolation_level = {}) {
return Accessor{this, override_isolation_level.value_or(isolation_level_)};
}
const std::string &LabelToName(LabelId label) const;
const std::string &PropertyToName(PropertyId property) const;
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const;
/// @throw std::bad_alloc if unable to insert a new mapping
LabelId NameToLabel(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
PropertyId NameToProperty(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
EdgeTypeId NameToEdgeType(std::string_view name);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, std::optional<uint64_t> desired_commit_timestamp = {});
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
bool DropIndex(LabelId label, std::optional<uint64_t> desired_commit_timestamp = {});
bool DropIndex(LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
IndicesInfo ListAllIndices() const;
/// Creates an existence constraint. Returns true if the constraint was
/// successfully added, false if it already exists and a `ConstraintViolation`
/// if there is an existing vertex violating the constraint.
///
/// @throw std::bad_alloc
/// @throw std::length_error
utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
/// Removes an existence constraint. Returns true if the constraint was
/// removed, and false if it doesn't exist.
bool DropExistenceConstraint(LabelId label, PropertyId property,
std::optional<uint64_t> desired_commit_timestamp = {});
/// Creates a unique constraint. In the case of two vertices violating the
/// constraint, it returns `ConstraintViolation`. Otherwise returns a
/// `UniqueConstraints::CreationStatus` enum with the following possibilities:
/// * `SUCCESS` if the constraint was successfully created,
/// * `ALREADY_EXISTS` if the constraint already existed,
/// * `EMPTY_PROPERTIES` if the property set is empty, or
// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the
// limit of maximum number of properties.
///
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> CreateUniqueConstraint(
LabelId label, const std::set<PropertyId> &properties, std::optional<uint64_t> desired_commit_timestamp = {});
/// Removes a unique constraint. Returns `UniqueConstraints::DeletionStatus`
/// enum with the following possibilities:
/// * `SUCCESS` if constraint was successfully removed,
/// * `NOT_FOUND` if the specified constraint was not found,
/// * `EMPTY_PROPERTIES` if the property set is empty, or
/// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the
// limit of maximum number of properties.
UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, const std::set<PropertyId> &properties,
std::optional<uint64_t> desired_commit_timestamp = {});
ConstraintsInfo ListAllConstraints() const;
SchemasInfo ListAllSchemas() const;
const Schemas::Schema *GetSchema(LabelId primary_label) const;
bool CreateSchema(LabelId primary_label, const std::vector<SchemaProperty> &schemas_types);
bool DropSchema(LabelId primary_label);
StorageInfo GetInfo() const;
bool LockPath();
bool UnlockPath();
bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config = {});
bool SetMainReplicationRole();
enum class RegisterReplicaError : uint8_t {
NAME_EXISTS,
END_POINT_EXISTS,
CONNECTION_FAILED,
COULD_NOT_BE_PERSISTED
};
/// @pre The instance should have a MAIN role
/// @pre Timeout can only be set for SYNC replication
utils::BasicResult<RegisterReplicaError, void> RegisterReplica(
std::string name, io::network::Endpoint endpoint, replication::ReplicationMode replication_mode,
const replication::ReplicationClientConfig &config = {});
/// @pre The instance should have a MAIN role
bool UnregisterReplica(std::string_view name);
std::optional<replication::ReplicaState> GetReplicaState(std::string_view name);
ReplicationRole GetReplicationRole() const;
struct ReplicaInfo {
std::string name;
replication::ReplicationMode mode;
std::optional<double> timeout;
io::network::Endpoint endpoint;
replication::ReplicaState state;
};
std::vector<ReplicaInfo> ReplicasInfo();
void FreeMemory();
void SetIsolationLevel(IsolationLevel isolation_level);
enum class CreateSnapshotError : uint8_t { DisabledForReplica };
utils::BasicResult<CreateSnapshotError> CreateSnapshot();
private:
Transaction CreateTransaction(IsolationLevel isolation_level);
/// The force parameter determines the behaviour of the garbage collector.
/// If it's set to true, it will behave as a global operation, i.e. it can't
/// be part of a transaction, and no other transaction can be active at the same time.
/// This allows it to delete immediately vertices without worrying that some other
/// transaction is possibly using it. If there are active transactions when this method
/// is called with force set to true, it will fallback to the same method with the force
/// set to false.
/// If it's set to false, it will execute in parallel with other transactions, ensuring
/// that no object in use can be deleted.
/// @throw std::system_error
/// @throw std::bad_alloc
template <bool force>
void CollectGarbage();
bool InitializeWalFile();
void FinalizeWalFile();
void AppendToWal(const Transaction &transaction, uint64_t final_commit_timestamp);
void AppendToWal(durability::StorageGlobalOperation operation, LabelId label, const std::set<PropertyId> &properties,
uint64_t final_commit_timestamp);
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});
[[nodiscard]] bool IsVertexBelongToShard(const VertexId &vertex_id) const;
// Main object storage
NameIdMapper name_id_mapper_;
LabelId primary_label_;
// The shard's range is [min, max)
PrimaryKey min_primary_key_;
std::optional<PrimaryKey> max_primary_key_;
VerticesSkipList vertices_;
utils::SkipList<Edge> edges_;
// Even though the edge count is already kept in the `edges_` SkipList, the
// list is used only when properties are enabled for edges. Because of that we
// keep a separate count of edges that is always updated.
uint64_t edge_count_{0};
SchemaValidator schema_validator_;
VertexValidator vertex_validator_;
Constraints constraints_;
Indices indices_;
Schemas schemas_;
// Transaction engine
uint64_t timestamp_{kTimestampInitialId};
uint64_t transaction_id_{kTransactionInitialId};
// TODO: This isn't really a commit log, it doesn't even care if a
// transaction commited or aborted. We could probably combine this with
// `timestamp_` in a sensible unit, something like TransactionClock or
// whatever.
std::optional<CommitLog> commit_log_;
std::list<Transaction> committed_transactions_;
IsolationLevel isolation_level_;
Config config_;
// Undo buffers that were unlinked and now are waiting to be freed.
std::list<std::pair<uint64_t, std::list<Delta>>> garbage_undo_buffers_;
// Vertices that are logically deleted but still have to be removed from
// indices before removing them from the main storage.
std::list<PrimaryKey> deleted_vertices_;
// Vertices that are logically deleted and removed from indices and now wait
// to be removed from the main storage.
std::list<std::pair<uint64_t, PrimaryKey>> garbage_vertices_;
// Edges that are logically deleted and wait to be removed from the main
// storage.
std::list<Gid> deleted_edges_;
// Durability
std::filesystem::path snapshot_directory_;
std::filesystem::path wal_directory_;
std::filesystem::path lock_file_path_;
utils::OutputFile lock_file_handle_;
// UUID used to distinguish snapshots and to link snapshots to WALs
std::string uuid_;
// Sequence number used to keep track of the chain of WALs.
uint64_t wal_seq_num_{0};
// UUID to distinguish different main instance runs for replication process
// on SAME storage.
// Multiple instances can have same storage UUID and be MAIN at the same time.
// We cannot compare commit timestamps of those instances if one of them
// becomes the replica of the other so we use epoch_id_ as additional
// discriminating property.
// Example of this:
// We have 2 instances of the same storage, S1 and S2.
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
// At the moment when S1 commited a transaction with timestamp 20, and S2
// a different transaction with timestamp 15, we change S2's role to REPLICA
// and register it on S1.
// Without using the epoch_id, we don't know that S1 and S2 have completely
// different transactions, we think that the S2 is behind only by 5 commits.
std::string epoch_id_;
// History of the previous epoch ids.
// Each value consists of the epoch id along the last commit belonging to that
// epoch.
std::deque<std::pair<std::string, uint64_t>> epoch_history_;
std::optional<durability::WalFile> wal_file_;
uint64_t wal_unsynced_transactions_{0};
utils::FileRetainer file_retainer_;
// Global locker that is used for clients file locking
utils::FileRetainer::FileLocker global_locker_;
// Last commited timestamp
uint64_t last_commit_timestamp_{kTimestampInitialId};
class ReplicationServer;
std::unique_ptr<ReplicationServer> replication_server_{nullptr};
class ReplicationClient;
// We create ReplicationClient using unique_ptr so we can move
// newly created client into the vector.
// We cannot move the client directly because it contains ThreadPool
// which cannot be moved. Also, the move is necessary because
// we don't want to create the client directly inside the vector
// because that would require the lock on the list putting all
// commits (they iterate list of clients) to halt.
// This way we can initialize client in main thread which means
// that we can immediately notify the user if the initialization
// failed.
using ReplicationClientList = utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>, utils::SpinLock>;
ReplicationClientList replication_clients_;
ReplicationRole replication_role_{ReplicationRole::MAIN};
};
} // namespace memgraph::storage::v3

359
src/storage/v3/shard_rsm.cpp Normal file
View File

@ -0,0 +1,359 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <iterator>
#include <utility>
#include "storage/v3/shard_rsm.hpp"
#include "storage/v3/value_conversions.hpp"
#include "storage/v3/vertex_accessor.hpp"
using memgraph::msgs::Label;
using memgraph::msgs::PropertyId;
using memgraph::msgs::Value;
using memgraph::msgs::VertexId;
using memgraph::storage::conversions::ConvertPropertyVector;
using memgraph::storage::conversions::ConvertValueVector;
using memgraph::storage::conversions::ToPropertyValue;
using memgraph::storage::conversions::ToValue;
namespace {
std::vector<std::pair<memgraph::storage::v3::PropertyId, memgraph::storage::v3::PropertyValue>> ConvertPropertyMap(
std::vector<std::pair<PropertyId, Value>> &&properties) {
std::vector<std::pair<memgraph::storage::v3::PropertyId, memgraph::storage::v3::PropertyValue>> ret;
ret.reserve(properties.size());
for (auto &[key, value] : properties) {
ret.emplace_back(std::make_pair(key, ToPropertyValue(std::move(value))));
}
return ret;
}
std::vector<std::pair<memgraph::storage::v3::PropertyId, Value>> FromMap(
const std::map<PropertyId, Value> &properties) {
std::vector<std::pair<memgraph::storage::v3::PropertyId, Value>> ret;
ret.reserve(properties.size());
for (const auto &[key, value] : properties) {
ret.emplace_back(std::make_pair(key, value));
}
return ret;
}
std::optional<std::map<PropertyId, Value>> CollectPropertiesFromAccessor(
const memgraph::storage::v3::VertexAccessor &acc, const std::vector<memgraph::storage::v3::PropertyId> &props,
memgraph::storage::v3::View view) {
std::map<PropertyId, Value> ret;
for (const auto &prop : props) {
auto result = acc.GetProperty(prop, view);
if (result.HasError()) {
spdlog::debug("Encountered an Error while trying to get a vertex property.");
continue;
}
auto &value = result.GetValue();
if (value.IsNull()) {
spdlog::debug("The specified property does not exist but it should");
continue;
}
ret.emplace(prop, ToValue(value));
}
return ret;
}
std::optional<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(
const memgraph::storage::v3::VertexAccessor &acc, memgraph::storage::v3::View view) {
std::map<PropertyId, Value> ret;
auto iter = acc.Properties(view);
if (iter.HasError()) {
spdlog::debug("Encountered an error while trying to get vertex properties.");
}
for (const auto &[prop_key, prop_val] : iter.GetValue()) {
ret.emplace(prop_key, ToValue(prop_val));
}
return ret;
}
Value ConstructValueVertex(const memgraph::storage::v3::VertexAccessor &acc, memgraph::storage::v3::View view) {
// Get the vertex id
auto prim_label = acc.PrimaryLabel(view).GetValue();
Label value_label{.id = prim_label};
auto prim_key = ConvertValueVector(acc.PrimaryKey(view).GetValue());
VertexId vertex_id = std::make_pair(value_label, prim_key);
// Get the labels
auto vertex_labels = acc.Labels(view).GetValue();
std::vector<Label> value_labels;
for (const auto &label : vertex_labels) {
Label l = {.id = label};
value_labels.push_back(l);
}
return Value({.id = vertex_id, .labels = value_labels});
}
} // namespace
namespace memgraph::storage::v3 {
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) {
auto acc = shard_->Access();
// Workaround untill we have access to CreateVertexAndValidate()
// with the new signature that does not require the primary label.
const auto prim_label = acc.GetPrimaryLabel();
bool action_successful = true;
for (auto &new_vertex : req.new_vertices) {
/// TODO(gvolfing) Remove this. In the new implementation each shard
/// should have a predetermined primary label, so there is no point in
/// specifying it in the accessor functions. Their signature will
/// change.
/// TODO(gvolfing) Consider other methods than converting. Change either
/// the way that the property map is stored in the messages, or the
/// signature of CreateVertexAndValidate.
auto converted_property_map = ConvertPropertyMap(std::move(new_vertex.properties));
// TODO(gvolfing) make sure if this conversion is actually needed.
std::vector<memgraph::storage::v3::LabelId> converted_label_ids;
converted_label_ids.reserve(new_vertex.label_ids.size());
for (const auto &label_id : new_vertex.label_ids) {
converted_label_ids.emplace_back(label_id.id);
}
auto result_schema =
acc.CreateVertexAndValidate(prim_label, converted_label_ids,
ConvertPropertyVector(std::move(new_vertex.primary_key)), converted_property_map);
if (result_schema.HasError()) {
auto &error = result_schema.GetError();
std::visit(
[]<typename T>(T &&) {
using ErrorType = std::remove_cvref_t<T>;
if constexpr (std::is_same_v<ErrorType, SchemaViolation>) {
spdlog::debug("Creating vertex failed with error: SchemaViolation");
} else if constexpr (std::is_same_v<ErrorType, Error>) {
spdlog::debug("Creating vertex failed with error: Error");
} else {
static_assert(kAlwaysFalse<T>, "Missing type from variant visitor");
}
},
error);
action_successful = false;
break;
}
}
msgs::CreateVerticesResponse resp{};
resp.success = action_successful;
if (action_successful) {
auto result = acc.Commit(req.transaction_id.logical_id);
if (result.HasError()) {
resp.success = false;
spdlog::debug(&"ConstraintViolation, commiting vertices was unsuccesfull with transaction id: "[req.transaction_id
.logical_id]);
}
}
return resp;
}
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteVerticesRequest &&req) {
bool action_successful = true;
auto acc = shard_->Access();
for (auto &propval : req.primary_keys) {
if (!action_successful) {
break;
}
auto vertex_acc = acc.FindVertex(ConvertPropertyVector(std::move(propval)), View::OLD);
if (!vertex_acc) {
spdlog::debug(
&"Error while trying to delete vertex. Vertex to delete does not exist. Transaction id: "[req.transaction_id
.logical_id]);
action_successful = false;
} else {
// TODO(gvolfing)
// Since we will not have different kinds of deletion types in one transaction,
// we dont have to enter the switch statement on every iteration. Optimize this.
switch (req.deletion_type) {
case msgs::DeleteVerticesRequest::DeletionType::DELETE: {
auto result = acc.DeleteVertex(&vertex_acc.value());
if (result.HasError() || !(result.GetValue().has_value())) {
action_successful = false;
spdlog::debug(&"Error while trying to delete vertex. Transaction id: "[req.transaction_id.logical_id]);
}
break;
}
case msgs::DeleteVerticesRequest::DeletionType::DETACH_DELETE: {
auto result = acc.DetachDeleteVertex(&vertex_acc.value());
if (result.HasError() || !(result.GetValue().has_value())) {
action_successful = false;
spdlog::debug(
&"Error while trying to detach and delete vertex. Transaction id: "[req.transaction_id.logical_id]);
}
break;
}
}
}
}
msgs::DeleteVerticesResponse resp{};
resp.success = action_successful;
if (action_successful) {
auto result = acc.Commit(req.transaction_id.logical_id);
if (result.HasError()) {
resp.success = false;
spdlog::debug(&"ConstraintViolation, commiting vertices was unsuccesfull with transaction id: "[req.transaction_id
.logical_id]);
}
}
return resp;
}
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateEdgesRequest &&req) {
auto acc = shard_->Access();
bool action_successful = true;
for (auto &edge : req.edges) {
auto vertex_acc_from_primary_key = edge.src.second;
auto vertex_from_acc = acc.FindVertex(ConvertPropertyVector(std::move(vertex_acc_from_primary_key)), View::OLD);
auto vertex_acc_to_primary_key = edge.dst.second;
auto vertex_to_acc = acc.FindVertex(ConvertPropertyVector(std::move(vertex_acc_to_primary_key)), View::OLD);
if (!vertex_from_acc || !vertex_to_acc) {
action_successful = false;
spdlog::debug(
&"Error while trying to insert edge, vertex does not exist. Transaction id: "[req.transaction_id.logical_id]);
break;
}
auto from_vertex_id = VertexId(edge.src.first.id, ConvertPropertyVector(std::move(edge.src.second)));
auto to_vertex_id = VertexId(edge.dst.first.id, ConvertPropertyVector(std::move(edge.dst.second)));
auto edge_acc =
acc.CreateEdge(from_vertex_id, to_vertex_id, EdgeTypeId::FromUint(edge.type.id), Gid::FromUint(edge.id.gid));
if (edge_acc.HasError()) {
action_successful = false;
spdlog::debug(&"Creating edge was not successful. Transaction id: "[req.transaction_id.logical_id]);
break;
}
}
msgs::CreateEdgesResponse resp{};
resp.success = action_successful;
if (action_successful) {
auto result = acc.Commit(req.transaction_id.logical_id);
if (result.HasError()) {
resp.success = false;
spdlog::debug(
&"ConstraintViolation, commiting edge creation was unsuccesfull with transaction id: "[req.transaction_id
.logical_id]);
}
}
return resp;
}
msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) {
auto acc = shard_->Access();
bool action_successful = true;
std::vector<msgs::ScanResultRow> results;
std::optional<msgs::VertexId> next_start_id;
const auto view = View(req.storage_view);
auto vertex_iterable = acc.Vertices(view);
bool did_reach_starting_point = false;
uint64_t sample_counter = 0;
for (auto it = vertex_iterable.begin(); it != vertex_iterable.end(); ++it) {
const auto &vertex = *it;
if (ConvertPropertyVector(std::move(req.start_id.second)) == vertex.PrimaryKey(View(req.storage_view)).GetValue()) {
did_reach_starting_point = true;
}
if (did_reach_starting_point) {
std::optional<std::map<PropertyId, Value>> found_props;
if (req.props_to_return) {
found_props = CollectPropertiesFromAccessor(vertex, req.props_to_return.value(), view);
} else {
found_props = CollectAllPropertiesFromAccessor(vertex, view);
}
if (!found_props) {
continue;
}
results.emplace_back(
msgs::ScanResultRow{.vertex = ConstructValueVertex(vertex, view), .props = FromMap(found_props.value())});
++sample_counter;
if (sample_counter == req.batch_limit) {
// Reached the maximum specified batch size.
// Get the next element before exiting.
const auto &next_vertex = *(++it);
next_start_id = ConstructValueVertex(next_vertex, view).vertex_v.id;
break;
}
}
}
msgs::ScanVerticesResponse resp{};
resp.success = action_successful;
if (action_successful) {
resp.next_start_id = next_start_id;
resp.results = std::move(results);
}
return resp;
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::UpdateVerticesRequest && /*req*/) {
return msgs::UpdateVerticesResponse{};
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::DeleteEdgesRequest && /*req*/) { return msgs::DeleteEdgesResponse{}; }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::UpdateEdgesRequest && /*req*/) { return msgs::UpdateEdgesResponse{}; }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::ReadResponses ShardRsm::HandleRead(msgs::ExpandOneRequest && /*req*/) { return msgs::ExpandOneResponse{}; }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest && /*req*/) {
return msgs::GetPropertiesResponse{};
}
} // namespace memgraph::storage::v3

58
src/storage/v3/shard_rsm.hpp Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <variant>
#include <openssl/ec.h>
#include "query/v2/requests.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
namespace memgraph::storage::v3 {
template <typename>
constexpr auto kAlwaysFalse = false;
class ShardRsm {
std::unique_ptr<Shard> shard_;
msgs::ReadResponses HandleRead(msgs::ExpandOneRequest &&req);
msgs::ReadResponses HandleRead(msgs::GetPropertiesRequest &&req);
msgs::ReadResponses HandleRead(msgs::ScanVerticesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::CreateVerticesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::DeleteVerticesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::UpdateVerticesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::CreateEdgesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::DeleteEdgesRequest &&req);
msgs::WriteResponses ApplyWrite(msgs::UpdateEdgesRequest &&req);
public:
explicit ShardRsm(std::unique_ptr<Shard> &&shard) : shard_(std::move(shard)){};
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::ReadResponses Read(msgs::ReadRequests requests) {
return std::visit([&](auto &&request) mutable { return HandleRead(std::forward<decltype(request)>(request)); },
std::move(requests));
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::WriteResponses Apply(msgs::WriteRequests requests) {
return std::visit([&](auto &&request) mutable { return ApplyWrite(std::forward<decltype(request)>(request)); },
std::move(requests));
}
};
} // namespace memgraph::storage::v3

2019
src/storage/v3/storage.cpp
View File

File diff suppressed because it is too large Load Diff

614
src/storage/v3/storage.hpp
View File

@ -11,622 +11,24 @@
#pragma once
#include <atomic>
#include <filesystem>
#include <optional>
#include <shared_mutex>
#include <variant>
#include <vector>
#include "io/network/endpoint.hpp"
#include "kvstore/kvstore.hpp"
#include "storage/v3/commit_log.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/constraints.hpp"
#include "storage/v3/durability/metadata.hpp"
#include "storage/v3/durability/wal.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/exceptions.hpp"
#include "utils/file_locker.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/rw_lock.hpp"
#include "utils/scheduler.hpp"
#include "utils/skip_list.hpp"
#include "utils/synchronized.hpp"
#include "utils/uuid.hpp"
#include <boost/asio/thread_pool.hpp>
/// REPLICATION ///
#include "rpc/server.hpp"
#include "storage/v3/replication/config.hpp"
#include "storage/v3/replication/enums.hpp"
#include "storage/v3/replication/rpc.hpp"
#include "storage/v3/replication/serialization.hpp"
#include "storage/v3/shard.hpp"
namespace memgraph::storage::v3 {
// The storage is based on this paper:
// https://db.in.tum.de/~muehlbau/papers/mvcc.pdf
// The paper implements a fully serializable storage, in our implementation we
// only implement snapshot isolation for transactions.
/// Iterable for iterating through all vertices of a Storage.
///
/// An instance of this will be usually be wrapped inside VerticesIterable for
/// generic, public use.
class AllVerticesIterable final {
utils::SkipList<Vertex>::Accessor vertices_accessor_;
Transaction *transaction_;
View view_;
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
const SchemaValidator *schema_validator_;
std::optional<VertexAccessor> vertex_;
class Storage {
public:
class Iterator final {
AllVerticesIterable *self_;
utils::SkipList<Vertex>::Iterator it_;
public:
Iterator(AllVerticesIterable *self, utils::SkipList<Vertex>::Iterator it);
VertexAccessor operator*() const;
Iterator &operator++();
bool operator==(const Iterator &other) const { return self_ == other.self_ && it_ == other.it_; }
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
AllVerticesIterable(utils::SkipList<Vertex>::Accessor vertices_accessor, Transaction *transaction, View view,
Indices *indices, Constraints *constraints, Config::Items config,
SchemaValidator *schema_validator)
: vertices_accessor_(std::move(vertices_accessor)),
transaction_(transaction),
view_(view),
indices_(indices),
constraints_(constraints),
config_(config),
schema_validator_(schema_validator) {}
Iterator begin() { return {this, vertices_accessor_.begin()}; }
Iterator end() { return {this, vertices_accessor_.end()}; }
};
/// Generic access to different kinds of vertex iterations.
///
/// This class should be the primary type used by the client code to iterate
/// over vertices inside a Storage instance.
class VerticesIterable final {
enum class Type { ALL, BY_LABEL, BY_LABEL_PROPERTY };
Type type_;
union {
AllVerticesIterable all_vertices_;
LabelIndex::Iterable vertices_by_label_;
LabelPropertyIndex::Iterable vertices_by_label_property_;
};
public:
explicit VerticesIterable(AllVerticesIterable);
explicit VerticesIterable(LabelIndex::Iterable);
explicit VerticesIterable(LabelPropertyIndex::Iterable);
VerticesIterable(const VerticesIterable &) = delete;
VerticesIterable &operator=(const VerticesIterable &) = delete;
VerticesIterable(VerticesIterable &&) noexcept;
VerticesIterable &operator=(VerticesIterable &&) noexcept;
~VerticesIterable();
class Iterator final {
Type type_;
union {
AllVerticesIterable::Iterator all_it_;
LabelIndex::Iterable::Iterator by_label_it_;
LabelPropertyIndex::Iterable::Iterator by_label_property_it_;
};
void Destroy() noexcept;
public:
explicit Iterator(AllVerticesIterable::Iterator);
explicit Iterator(LabelIndex::Iterable::Iterator);
explicit Iterator(LabelPropertyIndex::Iterable::Iterator);
Iterator(const Iterator &);
Iterator &operator=(const Iterator &);
Iterator(Iterator &&) noexcept;
Iterator &operator=(Iterator &&) noexcept;
~Iterator();
VertexAccessor operator*() const;
Iterator &operator++();
bool operator==(const Iterator &other) const;
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
Iterator begin();
Iterator end();
};
/// Structure used to return information about existing indices in the storage.
struct IndicesInfo {
std::vector<LabelId> label;
std::vector<std::pair<LabelId, PropertyId>> label_property;
};
/// Structure used to return information about existing constraints in the
/// storage.
struct ConstraintsInfo {
std::vector<std::pair<LabelId, PropertyId>> existence;
std::vector<std::pair<LabelId, std::set<PropertyId>>> unique;
};
/// Structure used to return information about existing schemas in the storage
struct SchemasInfo {
Schemas::SchemasList schemas;
};
/// Structure used to return information about the storage.
struct StorageInfo {
uint64_t vertex_count;
uint64_t edge_count;
double average_degree;
uint64_t memory_usage;
uint64_t disk_usage;
};
enum class ReplicationRole : uint8_t { MAIN, REPLICA };
class Storage final {
public:
/// @throw std::system_error
/// @throw std::bad_alloc
explicit Storage(Config config = Config());
~Storage();
class Accessor final {
private:
friend class Storage;
explicit Accessor(Storage *storage, IsolationLevel isolation_level);
public:
Accessor(const Accessor &) = delete;
Accessor &operator=(const Accessor &) = delete;
Accessor &operator=(Accessor &&other) = delete;
// NOTE: After the accessor is moved, all objects derived from it (accessors
// and iterators) are *invalid*. You have to get all derived objects again.
Accessor(Accessor &&other) noexcept;
~Accessor();
VertexAccessor CreateVertex();
VertexAccessor CreateVertex(Gid gid);
/// @throw std::bad_alloc
ResultSchema<VertexAccessor> CreateVertexAndValidate(
LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
std::optional<VertexAccessor> FindVertex(Gid gid, View view);
VerticesIterable Vertices(View view) {
return VerticesIterable(AllVerticesIterable(storage_->vertices_.access(), &transaction_, view,
&storage_->indices_, &storage_->constraints_, storage_->config_.items,
&storage_->schema_validator_));
}
VerticesIterable Vertices(LabelId label, View view);
VerticesIterable Vertices(LabelId label, PropertyId property, View view);
VerticesIterable Vertices(LabelId label, PropertyId property, const PropertyValue &value, View view);
VerticesIterable Vertices(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view);
/// Return approximate number of all vertices in the database.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount() const { return static_cast<int64_t>(storage_->vertices_.size()); }
/// Return approximate number of vertices with the given label.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label) const {
return storage_->indices_.label_index.ApproximateVertexCount(label);
}
/// Return approximate number of vertices with the given label and property.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property);
}
/// Return approximate number of vertices with the given label and the given
/// value for the given property. Note that this is always an over-estimate
/// and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
}
/// Return approximate number of vertices with the given label and value for
/// the given property in the range defined by provided upper and lower
/// bounds.
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const {
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
}
/// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
/// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
VertexAccessor *vertex);
/// @throw std::bad_alloc
Result<EdgeAccessor> CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type);
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
Result<std::optional<EdgeAccessor>> DeleteEdge(EdgeAccessor *edge);
const std::string &LabelToName(LabelId label) const;
const std::string &PropertyToName(PropertyId property) const;
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const;
/// @throw std::bad_alloc if unable to insert a new mapping
LabelId NameToLabel(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
PropertyId NameToProperty(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
EdgeTypeId NameToEdgeType(std::string_view name);
bool LabelIndexExists(LabelId label) const { return storage_->indices_.label_index.IndexExists(label); }
bool LabelPropertyIndexExists(LabelId label, PropertyId property) const {
return storage_->indices_.label_property_index.IndexExists(label, property);
}
IndicesInfo ListAllIndices() const {
return {storage_->indices_.label_index.ListIndices(), storage_->indices_.label_property_index.ListIndices()};
}
ConstraintsInfo ListAllConstraints() const {
return {ListExistenceConstraints(storage_->constraints_),
storage_->constraints_.unique_constraints.ListConstraints()};
}
const SchemaValidator &GetSchemaValidator() const;
SchemasInfo ListAllSchemas() const { return {storage_->schemas_.ListSchemas()}; }
void AdvanceCommand();
/// Commit returns `ConstraintViolation` if the changes made by this
/// transaction violate an existence or unique constraint. In that case the
/// transaction is automatically aborted. Otherwise, void is returned.
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, void> Commit(std::optional<uint64_t> desired_commit_timestamp = {});
/// @throw std::bad_alloc
void Abort();
void FinalizeTransaction();
private:
/// @throw std::bad_alloc
VertexAccessor CreateVertex(Gid gid, LabelId primary_label);
/// @throw std::bad_alloc
Result<EdgeAccessor> CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type, Gid gid);
Storage *storage_;
std::shared_lock<utils::RWLock> storage_guard_;
Transaction transaction_;
std::optional<uint64_t> commit_timestamp_;
bool is_transaction_active_;
Config::Items config_;
};
Accessor Access(std::optional<IsolationLevel> override_isolation_level = {}) {
return Accessor{this, override_isolation_level.value_or(isolation_level_)};
}
const std::string &LabelToName(LabelId label) const;
const std::string &PropertyToName(PropertyId property) const;
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const;
/// @throw std::bad_alloc if unable to insert a new mapping
LabelId NameToLabel(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
PropertyId NameToProperty(std::string_view name);
/// @throw std::bad_alloc if unable to insert a new mapping
EdgeTypeId NameToEdgeType(std::string_view name);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, std::optional<uint64_t> desired_commit_timestamp = {});
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
bool DropIndex(LabelId label, std::optional<uint64_t> desired_commit_timestamp = {});
bool DropIndex(LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
IndicesInfo ListAllIndices() const;
/// Creates an existence constraint. Returns true if the constraint was
/// successfully added, false if it already exists and a `ConstraintViolation`
/// if there is an existing vertex violating the constraint.
///
/// @throw std::bad_alloc
/// @throw std::length_error
utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp = {});
/// Removes an existence constraint. Returns true if the constraint was
/// removed, and false if it doesn't exist.
bool DropExistenceConstraint(LabelId label, PropertyId property,
std::optional<uint64_t> desired_commit_timestamp = {});
/// Creates a unique constraint. In the case of two vertices violating the
/// constraint, it returns `ConstraintViolation`. Otherwise returns a
/// `UniqueConstraints::CreationStatus` enum with the following possibilities:
/// * `SUCCESS` if the constraint was successfully created,
/// * `ALREADY_EXISTS` if the constraint already existed,
/// * `EMPTY_PROPERTIES` if the property set is empty, or
// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the
// limit of maximum number of properties.
///
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> CreateUniqueConstraint(
LabelId label, const std::set<PropertyId> &properties, std::optional<uint64_t> desired_commit_timestamp = {});
/// Removes a unique constraint. Returns `UniqueConstraints::DeletionStatus`
/// enum with the following possibilities:
/// * `SUCCESS` if constraint was successfully removed,
/// * `NOT_FOUND` if the specified constraint was not found,
/// * `EMPTY_PROPERTIES` if the property set is empty, or
/// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the
// limit of maximum number of properties.
UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, const std::set<PropertyId> &properties,
std::optional<uint64_t> desired_commit_timestamp = {});
ConstraintsInfo ListAllConstraints() const;
SchemasInfo ListAllSchemas() const;
const Schemas::Schema *GetSchema(LabelId primary_label) const;
bool CreateSchema(LabelId primary_label, const std::vector<SchemaProperty> &schemas_types);
bool DropSchema(LabelId primary_label);
StorageInfo GetInfo() const;
bool LockPath();
bool UnlockPath();
bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config = {});
bool SetMainReplicationRole();
enum class RegisterReplicaError : uint8_t {
NAME_EXISTS,
END_POINT_EXISTS,
CONNECTION_FAILED,
COULD_NOT_BE_PERSISTED
};
/// @pre The instance should have a MAIN role
/// @pre Timeout can only be set for SYNC replication
utils::BasicResult<RegisterReplicaError, void> RegisterReplica(
std::string name, io::network::Endpoint endpoint, replication::ReplicationMode replication_mode,
const replication::ReplicationClientConfig &config = {});
/// @pre The instance should have a MAIN role
bool UnregisterReplica(std::string_view name);
std::optional<replication::ReplicaState> GetReplicaState(std::string_view name);
ReplicationRole GetReplicationRole() const;
struct ReplicaInfo {
std::string name;
replication::ReplicationMode mode;
std::optional<double> timeout;
io::network::Endpoint endpoint;
replication::ReplicaState state;
};
std::vector<ReplicaInfo> ReplicasInfo();
void FreeMemory();
void SetIsolationLevel(IsolationLevel isolation_level);
enum class CreateSnapshotError : uint8_t { DisabledForReplica };
utils::BasicResult<CreateSnapshotError> CreateSnapshot();
explicit Storage(Config config);
// Interface toward shard manipulation
// Shard handler -> will use rsm client
private:
Transaction CreateTransaction(IsolationLevel isolation_level);
/// The force parameter determines the behaviour of the garbage collector.
/// If it's set to true, it will behave as a global operation, i.e. it can't
/// be part of a transaction, and no other transaction can be active at the same time.
/// This allows it to delete immediately vertices without worrying that some other
/// transaction is possibly using it. If there are active transactions when this method
/// is called with force set to true, it will fallback to the same method with the force
/// set to false.
/// If it's set to false, it will execute in parallel with other transactions, ensuring
/// that no object in use can be deleted.
/// @throw std::system_error
/// @throw std::bad_alloc
template <bool force>
void CollectGarbage();
bool InitializeWalFile();
void FinalizeWalFile();
void AppendToWal(const Transaction &transaction, uint64_t final_commit_timestamp);
void AppendToWal(durability::StorageGlobalOperation operation, LabelId label, const std::set<PropertyId> &properties,
uint64_t final_commit_timestamp);
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});
// Main storage lock.
//
// Accessors take a shared lock when starting, so it is possible to block
// creation of new accessors by taking a unique lock. This is used when doing
// operations on storage that affect the global state, for example index
// creation.
mutable utils::RWLock main_lock_{utils::RWLock::Priority::WRITE};
// Main object storage
utils::SkipList<Vertex> vertices_;
utils::SkipList<Edge> edges_;
std::atomic<uint64_t> vertex_id_{0};
std::atomic<uint64_t> edge_id_{0};
// Even though the edge count is already kept in the `edges_` SkipList, the
// list is used only when properties are enabled for edges. Because of that we
// keep a separate count of edges that is always updated.
std::atomic<uint64_t> edge_count_{0};
NameIdMapper name_id_mapper_;
SchemaValidator schema_validator_;
Constraints constraints_;
Indices indices_;
Schemas schemas_;
// Transaction engine
utils::SpinLock engine_lock_;
uint64_t timestamp_{kTimestampInitialId};
uint64_t transaction_id_{kTransactionInitialId};
// TODO: This isn't really a commit log, it doesn't even care if a
// transaction commited or aborted. We could probably combine this with
// `timestamp_` in a sensible unit, something like TransactionClock or
// whatever.
std::optional<CommitLog> commit_log_;
utils::Synchronized<std::list<Transaction>, utils::SpinLock> committed_transactions_;
IsolationLevel isolation_level_;
std::vector<Shard> shards_;
boost::asio::thread_pool shard_handlers_;
Config config_;
utils::Scheduler gc_runner_;
std::mutex gc_lock_;
// Undo buffers that were unlinked and now are waiting to be freed.
utils::Synchronized<std::list<std::pair<uint64_t, std::list<Delta>>>, utils::SpinLock> garbage_undo_buffers_;
// Vertices that are logically deleted but still have to be removed from
// indices before removing them from the main storage.
utils::Synchronized<std::list<Gid>, utils::SpinLock> deleted_vertices_;
// Vertices that are logically deleted and removed from indices and now wait
// to be removed from the main storage.
std::list<std::pair<uint64_t, Gid>> garbage_vertices_;
// Edges that are logically deleted and wait to be removed from the main
// storage.
utils::Synchronized<std::list<Gid>, utils::SpinLock> deleted_edges_;
// Durability
std::filesystem::path snapshot_directory_;
std::filesystem::path wal_directory_;
std::filesystem::path lock_file_path_;
utils::OutputFile lock_file_handle_;
utils::Scheduler snapshot_runner_;
utils::SpinLock snapshot_lock_;
// UUID used to distinguish snapshots and to link snapshots to WALs
std::string uuid_;
// Sequence number used to keep track of the chain of WALs.
uint64_t wal_seq_num_{0};
// UUID to distinguish different main instance runs for replication process
// on SAME storage.
// Multiple instances can have same storage UUID and be MAIN at the same time.
// We cannot compare commit timestamps of those instances if one of them
// becomes the replica of the other so we use epoch_id_ as additional
// discriminating property.
// Example of this:
// We have 2 instances of the same storage, S1 and S2.
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
// At the moment when S1 commited a transaction with timestamp 20, and S2
// a different transaction with timestamp 15, we change S2's role to REPLICA
// and register it on S1.
// Without using the epoch_id, we don't know that S1 and S2 have completely
// different transactions, we think that the S2 is behind only by 5 commits.
std::string epoch_id_;
// History of the previous epoch ids.
// Each value consists of the epoch id along the last commit belonging to that
// epoch.
std::deque<std::pair<std::string, uint64_t>> epoch_history_;
std::optional<durability::WalFile> wal_file_;
uint64_t wal_unsynced_transactions_{0};
utils::FileRetainer file_retainer_;
// Global locker that is used for clients file locking
utils::FileRetainer::FileLocker global_locker_;
// Last commited timestamp
std::atomic<uint64_t> last_commit_timestamp_{kTimestampInitialId};
class ReplicationServer;
std::unique_ptr<ReplicationServer> replication_server_{nullptr};
class ReplicationClient;
// We create ReplicationClient using unique_ptr so we can move
// newly created client into the vector.
// We cannot move the client directly because it contains ThreadPool
// which cannot be moved. Also, the move is necessary because
// we don't want to create the client directly inside the vector
// because that would require the lock on the list putting all
// commits (they iterate list of clients) to halt.
// This way we can initialize client in main thread which means
// that we can immediately notify the user if the initialization
// failed.
using ReplicationClientList = utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>, utils::SpinLock>;
ReplicationClientList replication_clients_;
std::atomic<ReplicationRole> replication_role_{ReplicationRole::MAIN};
};
} // namespace memgraph::storage::v3

131
src/storage/v3/value_conversions.hpp Normal file
View File

@ -0,0 +1,131 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/requests.hpp"
#include "storage/v3/property_value.hpp"
#include "utils/logging.hpp"
#include <map>
#include <string>
#include <utility>
#include <vector>
#pragma once
// TODO(kostasrim) Think about long term sustainability
// This should not be put under v3 because ADL will mess that up.
namespace memgraph::storage::conversions {
using memgraph::msgs::Label;
using memgraph::msgs::PropertyId;
using memgraph::msgs::Value;
using memgraph::msgs::VertexId;
// TODO(gvolfing use come algorithm instead of explicit for loops)
inline memgraph::storage::v3::PropertyValue ToPropertyValue(Value value) {
using PV = memgraph::storage::v3::PropertyValue;
PV ret;
switch (value.type) {
case Value::Type::Null:
return PV{};
case Value::Type::Bool:
return PV(value.bool_v);
case Value::Type::Int64:
return PV(static_cast<int64_t>(value.int_v));
case Value::Type::Double:
return PV(value.double_v);
case Value::Type::String:
return PV(value.string_v);
case Value::Type::List: {
std::vector<PV> list;
for (auto &elem : value.list_v) {
list.emplace_back(ToPropertyValue(std::move(elem)));
}
return PV(list);
}
case Value::Type::Map: {
std::map<std::string, PV> map;
for (auto &[key, value] : value.map_v) {
map.emplace(std::make_pair(key, ToPropertyValue(std::move(value))));
}
return PV(map);
}
// These are not PropertyValues
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::Path:
MG_ASSERT(false, "Not PropertyValue");
}
return ret;
}
inline Value ToValue(const memgraph::storage::v3::PropertyValue &pv) {
using memgraph::storage::v3::PropertyValue;
switch (pv.type()) {
case PropertyValue::Type::Bool:
return Value(pv.ValueBool());
case PropertyValue::Type::Double:
return Value(pv.ValueDouble());
case PropertyValue::Type::Int:
return Value(pv.ValueInt());
case PropertyValue::Type::List: {
std::vector<Value> list(pv.ValueList().size());
for (const auto &elem : pv.ValueList()) {
list.emplace_back(ToValue(elem));
}
return Value(list);
}
case PropertyValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &[key, val] : pv.ValueMap()) {
// maybe use std::make_pair once the && issue is resolved.
map.emplace(key, ToValue(val));
}
return Value(map);
}
case PropertyValue::Type::Null:
return Value{};
case PropertyValue::Type::String:
return Value(pv.ValueString());
case PropertyValue::Type::TemporalData: {
// TBD -> we need to specify this in the messages, not a priority.
MG_ASSERT(false, "Temporal datatypes are not yet implemented on Value!");
return Value{};
}
}
}
inline std::vector<memgraph::storage::v3::PropertyValue> ConvertPropertyVector(std::vector<Value> vec) {
std::vector<memgraph::storage::v3::PropertyValue> ret;
ret.reserve(vec.size());
for (auto &elem : vec) {
ret.push_back(ToPropertyValue(std::move(elem)));
}
return ret;
}
inline std::vector<Value> ConvertValueVector(const std::vector<memgraph::storage::v3::PropertyValue> &vec) {
std::vector<Value> ret;
ret.reserve(vec.size());
for (const auto &elem : vec) {
ret.push_back(ToValue(elem));
}
return ret;
}
} // namespace memgraph::storage::conversions

47
src/storage/v3/vertex.hpp
View File

@ -13,53 +13,39 @@
#include <limits>
#include <tuple>
#include <type_traits>
#include <vector>
#include "storage/v3/delta.hpp"
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex_id.hpp"
#include "utils/algorithm.hpp"
#include "utils/spin_lock.hpp"
namespace memgraph::storage::v3 {
struct Vertex {
Vertex(Gid gid, Delta *delta, LabelId primary_label)
: gid(gid), primary_label{primary_label}, deleted(false), delta(delta) {
using EdgeLink = std::tuple<EdgeTypeId, VertexId, EdgeRef>;
Vertex(Delta *delta, const std::vector<PropertyValue> &primary_properties) : keys{primary_properties}, delta{delta} {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
"Vertex must be created with an initial DELETE_OBJECT delta!");
}
// TODO remove this when import replication is solved
Vertex(Gid gid, LabelId primary_label) : gid(gid), primary_label{primary_label}, deleted(false) {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
"Vertex must be created with an initial DELETE_OBJECT delta!");
}
friend bool operator==(const Vertex &vertex, const PrimaryKey &primary_key) { return vertex.keys == primary_key; }
// TODO remove this when import csv is solved
Vertex(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
"Vertex must be created with an initial DELETE_OBJECT delta!");
}
KeyStore keys;
// TODO remove this when import replication is solved
explicit Vertex(Gid gid) : gid(gid), deleted(false) {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
"Vertex must be created with an initial DELETE_OBJECT delta!");
}
Gid gid;
LabelId primary_label;
std::vector<LabelId> labels;
PropertyStore properties;
std::vector<EdgeLink> in_edges;
std::vector<EdgeLink> out_edges;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> in_edges;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> out_edges;
mutable utils::SpinLock lock;
bool deleted;
bool deleted{false};
// uint8_t PAD;
// uint16_t PAD;
@ -68,13 +54,6 @@ struct Vertex {
static_assert(alignof(Vertex) >= 8, "The Vertex should be aligned to at least 8!");
inline bool operator==(const Vertex &first, const Vertex &second) { return first.gid == second.gid; }
inline bool operator<(const Vertex &first, const Vertex &second) { return first.gid < second.gid; }
inline bool operator==(const Vertex &first, const Gid &second) { return first.gid == second; }
inline bool operator<(const Vertex &first, const Gid &second) { return first.gid < second; }
inline bool VertexHasLabel(const Vertex &vertex, const LabelId label) {
return vertex.primary_label == label || utils::Contains(vertex.labels, label);
}
inline bool VertexHasLabel(const Vertex &vertex, const LabelId label) { return utils::Contains(vertex.labels, label); }
} // namespace memgraph::storage::v3

195
src/storage/v3/vertex_accessor.cpp
View File

@ -16,6 +16,7 @@
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schema_validator.hpp"
@ -32,7 +33,6 @@ std::pair<bool, bool> IsVisible(Vertex *vertex, Transaction *transaction, View v
bool deleted = false;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex->lock);
deleted = vertex->deleted;
delta = vertex->delta;
}
@ -64,12 +64,12 @@ std::pair<bool, bool> IsVisible(Vertex *vertex, Transaction *transaction, View v
std::optional<VertexAccessor> VertexAccessor::Create(Vertex *vertex, Transaction *transaction, Indices *indices,
Constraints *constraints, Config::Items config,
const SchemaValidator &schema_validator, View view) {
const VertexValidator &vertex_validator, View view) {
if (const auto [exists, deleted] = detail::IsVisible(vertex, transaction, view); !exists || deleted) {
return std::nullopt;
}
return VertexAccessor{vertex, transaction, indices, constraints, config, schema_validator};
return VertexAccessor{vertex, transaction, indices, constraints, config, vertex_validator};
}
bool VertexAccessor::IsVisible(View view) const {
@ -79,7 +79,6 @@ bool VertexAccessor::IsVisible(View view) const {
Result<bool> VertexAccessor::AddLabel(LabelId label) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
@ -97,11 +96,10 @@ Result<bool> VertexAccessor::AddLabel(LabelId label) {
}
ResultSchema<bool> VertexAccessor::AddLabelAndValidate(LabelId label) {
if (const auto maybe_violation_error = vertex_validator_.ValidateAddLabel(label); maybe_violation_error) {
if (const auto maybe_violation_error = vertex_validator_->ValidateAddLabel(label); maybe_violation_error) {
return {*maybe_violation_error};
}
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return {Error::SERIALIZATION_ERROR};
@ -119,8 +117,6 @@ ResultSchema<bool> VertexAccessor::AddLabelAndValidate(LabelId label) {
}
Result<bool> VertexAccessor::RemoveLabel(LabelId label) {
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
if (vertex_->deleted) return Error::DELETED_OBJECT;
@ -136,10 +132,9 @@ Result<bool> VertexAccessor::RemoveLabel(LabelId label) {
}
ResultSchema<bool> VertexAccessor::RemoveLabelAndValidate(LabelId label) {
if (const auto maybe_violation_error = vertex_validator_.ValidateRemoveLabel(label); maybe_violation_error) {
if (const auto maybe_violation_error = vertex_validator_->ValidateRemoveLabel(label); maybe_violation_error) {
return {*maybe_violation_error};
}
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return {Error::SERIALIZATION_ERROR};
@ -161,9 +156,8 @@ Result<bool> VertexAccessor::HasLabel(LabelId label, View view) const {
bool has_label = false;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
has_label = VertexHasLabel(*vertex_, label);
has_label = label == vertex_validator_->primary_label_ || VertexHasLabel(*vertex_, label);
delta = vertex_->delta;
}
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &has_label, label](const Delta &delta) {
@ -204,46 +198,33 @@ Result<bool> VertexAccessor::HasLabel(LabelId label, View view) const {
}
Result<LabelId> VertexAccessor::PrimaryLabel(const View view) const {
bool exists = true;
bool deleted = false;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
delta = vertex_->delta;
if (const auto result = CheckVertexExistence(view); result.HasError()) {
return result.GetError();
}
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted](const Delta &delta) {
switch (delta.action) {
case Delta::Action::DELETE_OBJECT: {
exists = false;
break;
}
case Delta::Action::RECREATE_OBJECT: {
deleted = false;
break;
}
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL:
case Delta::Action::SET_PROPERTY:
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::REMOVE_OUT_EDGE:
break;
}
});
if (!exists) return Error::NONEXISTENT_OBJECT;
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
return vertex_->primary_label;
return vertex_validator_->primary_label_;
}
Result<PrimaryKey> VertexAccessor::PrimaryKey(const View view) const {
if (const auto result = CheckVertexExistence(view); result.HasError()) {
return result.GetError();
}
return vertex_->keys.Keys();
}
Result<VertexId> VertexAccessor::Id(View view) const {
if (const auto result = CheckVertexExistence(view); result.HasError()) {
return result.GetError();
}
return VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
};
Result<std::vector<LabelId>> VertexAccessor::Labels(View view) const {
bool exists = true;
bool deleted = false;
std::vector<LabelId> labels;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
labels = vertex_->labels;
delta = vertex_->delta;
@ -288,7 +269,6 @@ Result<std::vector<LabelId>> VertexAccessor::Labels(View view) const {
Result<PropertyValue> VertexAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
@ -309,12 +289,48 @@ Result<PropertyValue> VertexAccessor::SetProperty(PropertyId property, const Pro
return std::move(current_value);
}
Result<void> VertexAccessor::CheckVertexExistence(View view) const {
bool exists = true;
bool deleted = false;
Delta *delta = nullptr;
{
deleted = vertex_->deleted;
delta = vertex_->delta;
}
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted](const Delta &delta) {
switch (delta.action) {
case Delta::Action::DELETE_OBJECT: {
exists = false;
break;
}
case Delta::Action::RECREATE_OBJECT: {
deleted = false;
break;
}
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL:
case Delta::Action::SET_PROPERTY:
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::REMOVE_OUT_EDGE:
break;
}
});
if (!exists) {
return Error::NONEXISTENT_OBJECT;
}
if (!for_deleted_ && deleted) {
return Error::DELETED_OBJECT;
}
return {};
}
ResultSchema<PropertyValue> VertexAccessor::SetPropertyAndValidate(PropertyId property, const PropertyValue &value) {
if (auto maybe_violation_error = vertex_validator_.ValidatePropertyUpdate(property); maybe_violation_error) {
if (auto maybe_violation_error = vertex_validator_->ValidatePropertyUpdate(property); maybe_violation_error) {
return {*maybe_violation_error};
}
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) {
return {Error::SERIALIZATION_ERROR};
@ -340,8 +356,6 @@ ResultSchema<PropertyValue> VertexAccessor::SetPropertyAndValidate(PropertyId pr
}
Result<std::map<PropertyId, PropertyValue>> VertexAccessor::ClearProperties() {
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
if (vertex_->deleted) return Error::DELETED_OBJECT;
@ -363,7 +377,6 @@ Result<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View view
PropertyValue value;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
value = vertex_->properties.GetProperty(property);
delta = vertex_->delta;
@ -404,7 +417,6 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view
std::map<PropertyId, PropertyValue> properties;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
properties = vertex_->properties.Properties();
delta = vertex_->delta;
@ -449,21 +461,21 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view
}
Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const std::vector<EdgeTypeId> &edge_types,
const VertexAccessor *destination) const {
MG_ASSERT(!destination || destination->transaction_ == transaction_, "Invalid accessor!");
const VertexId *destination_id) const {
bool exists = true;
bool deleted = false;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> in_edges;
std::vector<Vertex::EdgeLink> in_edges;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
if (edge_types.empty() && !destination) {
if (edge_types.empty() && nullptr == destination_id) {
in_edges = vertex_->in_edges;
} else {
for (const auto &item : vertex_->in_edges) {
const auto &[edge_type, from_vertex, edge] = item;
if (destination && from_vertex != destination->vertex_) continue;
if (nullptr != destination_id && from_vertex != *destination_id) {
continue;
};
if (!edge_types.empty() && std::find(edge_types.begin(), edge_types.end(), edge_type) == edge_types.end())
continue;
in_edges.push_back(item);
@ -472,29 +484,27 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const std::
delta = vertex_->delta;
}
ApplyDeltasForRead(
transaction_, delta, view, [&exists, &deleted, &in_edges, &edge_types, &destination](const Delta &delta) {
transaction_, delta, view, [&exists, &deleted, &in_edges, &edge_types, destination_id](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_IN_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_) break;
if (nullptr != destination_id && delta.vertex_edge.vertex_id != *destination_id) break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
break;
// Add the edge because we don't see the removal.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
delta.vertex_edge.edge};
Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
auto it = std::find(in_edges.begin(), in_edges.end(), link);
MG_ASSERT(it == in_edges.end(), "Invalid database state!");
in_edges.push_back(link);
break;
}
case Delta::Action::REMOVE_IN_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_) break;
if (nullptr != destination_id && delta.vertex_edge.vertex_id != *destination_id) break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
break;
// Remove the label because we don't see the addition.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
delta.vertex_edge.edge};
Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
auto it = std::find(in_edges.begin(), in_edges.end(), link);
MG_ASSERT(it != in_edges.end(), "Invalid database state!");
std::swap(*it, *in_edges.rbegin());
@ -520,31 +530,32 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const std::
if (!exists) return Error::NONEXISTENT_OBJECT;
if (deleted) return Error::DELETED_OBJECT;
std::vector<EdgeAccessor> ret;
if (in_edges.empty()) {
return ret;
}
ret.reserve(in_edges.size());
const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
for (const auto &item : in_edges) {
const auto &[edge_type, from_vertex, edge] = item;
ret.emplace_back(edge, edge_type, from_vertex, vertex_, transaction_, indices_, constraints_, config_,
*vertex_validator_.schema_validator);
ret.emplace_back(edge, edge_type, from_vertex, id, transaction_, indices_, constraints_, config_);
}
return std::move(ret);
return ret;
}
Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const std::vector<EdgeTypeId> &edge_types,
const VertexAccessor *destination) const {
MG_ASSERT(!destination || destination->transaction_ == transaction_, "Invalid accessor!");
const VertexId *destination_id) const {
bool exists = true;
bool deleted = false;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> out_edges;
std::vector<Vertex::EdgeLink> out_edges;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
if (edge_types.empty() && !destination) {
if (edge_types.empty() && nullptr == destination_id) {
out_edges = vertex_->out_edges;
} else {
for (const auto &item : vertex_->out_edges) {
const auto &[edge_type, to_vertex, edge] = item;
if (destination && to_vertex != destination->vertex_) continue;
if (nullptr != destination_id && to_vertex != *destination_id) continue;
if (!edge_types.empty() && std::find(edge_types.begin(), edge_types.end(), edge_type) == edge_types.end())
continue;
out_edges.push_back(item);
@ -553,29 +564,27 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const std:
delta = vertex_->delta;
}
ApplyDeltasForRead(
transaction_, delta, view, [&exists, &deleted, &out_edges, &edge_types, &destination](const Delta &delta) {
transaction_, delta, view, [&exists, &deleted, &out_edges, &edge_types, destination_id](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_OUT_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_) break;
if (nullptr != destination_id && delta.vertex_edge.vertex_id != *destination_id) break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
break;
// Add the edge because we don't see the removal.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
delta.vertex_edge.edge};
Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
auto it = std::find(out_edges.begin(), out_edges.end(), link);
MG_ASSERT(it == out_edges.end(), "Invalid database state!");
out_edges.push_back(link);
break;
}
case Delta::Action::REMOVE_OUT_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_) break;
if (nullptr != destination_id && delta.vertex_edge.vertex_id != *destination_id) break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
break;
// Remove the label because we don't see the addition.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
delta.vertex_edge.edge};
Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
auto it = std::find(out_edges.begin(), out_edges.end(), link);
MG_ASSERT(it != out_edges.end(), "Invalid database state!");
std::swap(*it, *out_edges.rbegin());
@ -601,13 +610,16 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const std:
if (!exists) return Error::NONEXISTENT_OBJECT;
if (deleted) return Error::DELETED_OBJECT;
std::vector<EdgeAccessor> ret;
if (out_edges.empty()) {
return ret;
}
ret.reserve(out_edges.size());
const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
for (const auto &item : out_edges) {
const auto &[edge_type, to_vertex, edge] = item;
ret.emplace_back(edge, edge_type, vertex_, to_vertex, transaction_, indices_, constraints_, config_,
*vertex_validator_.schema_validator);
ret.emplace_back(edge, edge_type, id, to_vertex, transaction_, indices_, constraints_, config_);
}
return std::move(ret);
return ret;
}
Result<size_t> VertexAccessor::InDegree(View view) const {
@ -616,7 +628,6 @@ Result<size_t> VertexAccessor::InDegree(View view) const {
size_t degree = 0;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
degree = vertex_->in_edges.size();
delta = vertex_->delta;
@ -654,7 +665,6 @@ Result<size_t> VertexAccessor::OutDegree(View view) const {
size_t degree = 0;
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
degree = vertex_->out_edges.size();
delta = vertex_->delta;
@ -686,21 +696,4 @@ Result<size_t> VertexAccessor::OutDegree(View view) const {
return degree;
}
VertexAccessor::VertexValidator::VertexValidator(const SchemaValidator &schema_validator, const Vertex *vertex)
: schema_validator{&schema_validator}, vertex_{vertex} {}
[[nodiscard]] std::optional<SchemaViolation> VertexAccessor::VertexValidator::ValidatePropertyUpdate(
PropertyId property_id) const {
MG_ASSERT(vertex_ != nullptr, "Cannot validate vertex which is nullptr");
return schema_validator->ValidatePropertyUpdate(vertex_->primary_label, property_id);
};
[[nodiscard]] std::optional<SchemaViolation> VertexAccessor::VertexValidator::ValidateAddLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}
[[nodiscard]] std::optional<SchemaViolation> VertexAccessor::VertexValidator::ValidateRemoveLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}
} // namespace memgraph::storage::v3

76
src/storage/v3/vertex_accessor.hpp
View File

@ -13,77 +13,53 @@
#include <optional>
#include "storage/v3/id_types.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
class Storage;
class Shard;
struct Indices;
struct Constraints;
class VertexAccessor final {
private:
struct VertexValidator {
// TODO(jbajic) Beware since vertex is pointer it will be accessed even as nullptr
explicit VertexValidator(const SchemaValidator &schema_validator, const Vertex *vertex);
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(PropertyId property_id) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateAddLabel(LabelId label) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateRemoveLabel(LabelId label) const;
const SchemaValidator *schema_validator;
private:
const Vertex *vertex_;
};
friend class Storage;
friend class Shard;
public:
// Be careful when using VertexAccessor since it can be instantiated with
// nullptr values
VertexAccessor(Vertex *vertex, Transaction *transaction, Indices *indices, Constraints *constraints,
Config::Items config, const SchemaValidator &schema_validator, bool for_deleted = false)
Config::Items config, const VertexValidator &vertex_validator, bool for_deleted = false)
: vertex_(vertex),
transaction_(transaction),
indices_(indices),
constraints_(constraints),
config_(config),
vertex_validator_{schema_validator, vertex},
vertex_validator_{&vertex_validator},
for_deleted_(for_deleted) {}
static std::optional<VertexAccessor> Create(Vertex *vertex, Transaction *transaction, Indices *indices,
Constraints *constraints, Config::Items config,
const SchemaValidator &schema_validator, View view);
const VertexValidator &vertex_validator, View view);
/// @return true if the object is visible from the current transaction
bool IsVisible(View view) const;
/// Add a label and return `true` if insertion took place.
/// `false` is returned if the label already existed.
/// @throw std::bad_alloc
Result<bool> AddLabel(LabelId label);
/// Add a label and return `true` if insertion took place.
/// `false` is returned if the label already existed, or SchemaViolation
/// if adding the label has violated one of the schema constraints.
/// @throw std::bad_alloc
ResultSchema<bool> AddLabelAndValidate(LabelId label);
/// Remove a label and return `true` if deletion took place.
/// `false` is returned if the vertex did not have a label already.
/// @throw std::bad_alloc
Result<bool> RemoveLabel(LabelId label);
/// Remove a label and return `true` if deletion took place.
/// `false` is returned if the vertex did not have a label already. or SchemaViolation
/// if adding the label has violated one of the schema constraints.
@ -99,9 +75,9 @@ class VertexAccessor final {
Result<LabelId> PrimaryLabel(View view) const;
/// Set a property value and return the old value.
/// @throw std::bad_alloc
Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
Result<PrimaryKey> PrimaryKey(View view) const;
Result<VertexId> Id(View view) const;
/// Set a property value and return the old value or error.
/// @throw std::bad_alloc
@ -121,20 +97,18 @@ class VertexAccessor final {
/// @throw std::length_error if the resulting vector exceeds
/// std::vector::max_size().
Result<std::vector<EdgeAccessor>> InEdges(View view, const std::vector<EdgeTypeId> &edge_types = {},
const VertexAccessor *destination = nullptr) const;
const VertexId *destination_id = nullptr) const;
/// @throw std::bad_alloc
/// @throw std::length_error if the resulting vector exceeds
/// std::vector::max_size().
Result<std::vector<EdgeAccessor>> OutEdges(View view, const std::vector<EdgeTypeId> &edge_types = {},
const VertexAccessor *destination = nullptr) const;
const VertexId *destination_id = nullptr) const;
Result<size_t> InDegree(View view) const;
Result<size_t> OutDegree(View view) const;
Gid Gid() const noexcept { return vertex_->gid; }
const SchemaValidator *GetSchemaValidator() const;
bool operator==(const VertexAccessor &other) const noexcept {
@ -143,12 +117,28 @@ class VertexAccessor final {
bool operator!=(const VertexAccessor &other) const noexcept { return !(*this == other); }
private:
/// Add a label and return `true` if insertion took place.
/// `false` is returned if the label already existed.
/// @throw std::bad_alloc
Result<bool> AddLabel(LabelId label);
/// Remove a label and return `true` if deletion took place.
/// `false` is returned if the vertex did not have a label already.
/// @throw std::bad_alloc
Result<bool> RemoveLabel(LabelId label);
/// Set a property value and return the old value.
/// @throw std::bad_alloc
Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
Result<void> CheckVertexExistence(View view) const;
Vertex *vertex_;
Transaction *transaction_;
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
VertexValidator vertex_validator_;
const VertexValidator *vertex_validator_;
// if the accessor was created for a deleted vertex.
// Accessor behaves differently for some methods based on this
@ -165,6 +155,6 @@ class VertexAccessor final {
namespace std {
template <>
struct hash<memgraph::storage::v3::VertexAccessor> {
size_t operator()(const memgraph::storage::v3::VertexAccessor &v) const noexcept { return v.Gid().AsUint(); }
size_t operator()(const memgraph::storage::v3::VertexAccessor & /*v*/) const noexcept { return 0; }
};
} // namespace std

32
src/storage/v3/vertex_id.hpp Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
namespace memgraph::storage::v3 {
// TODO(antaljanosbenjamin): It is possible to use a union of the current primary key and a vertex pointer: for local
// vertices we can spare some space by eliminating copying the primary label and key, however it might introduce some
// overhead for "remove vertices", because of the extra enum that is necessary for this optimization.
struct VertexId {
VertexId(const LabelId primary_label, PrimaryKey primary_key)
: primary_label{primary_label}, primary_key{std::move(primary_key)} {}
LabelId primary_label;
PrimaryKey primary_key;
};
inline bool operator==(const VertexId &lhs, const VertexId &rhs) {
return lhs.primary_label == rhs.primary_label && lhs.primary_key == rhs.primary_key;
}
} // namespace memgraph::storage::v3

19
src/storage/v3/vertices_skip_list.hpp Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::storage::v3 {
using VerticesSkipList = utils::SkipList<LexicographicallyOrderedVertex>;
} // namespace memgraph::storage::v3

16
src/utils/concepts.hpp
View File

@ -11,6 +11,7 @@
#pragma once
#include <concepts>
#include <iterator>
namespace memgraph::utils {
template <typename T, typename... Args>
@ -18,4 +19,19 @@ concept SameAsAnyOf = (std::same_as<T, Args> || ...);
template <typename T>
concept Enum = std::is_enum_v<T>;
// WithRef, CanReference and Dereferenceable is based on the similarly named concepts in GCC 11.2.0
// bits/iterator_concepts.h
template <typename T>
using WithRef = T &;
template <typename T>
concept CanReference = requires {
typename WithRef<T>;
};
template <typename T>
concept Dereferenceable = requires(T t) {
{ *t } -> CanReference;
};
} // namespace memgraph::utils

12
tests/simulation/CMakeLists.txt
View File

@ -1,36 +1,34 @@
set(test_prefix memgraph__simulation__)
find_package(gflags)
find_package(Boost REQUIRED)
find_package(OpenSSL REQUIRED)
add_custom_target(memgraph__simulation)
function(add_simulation_test test_cpp san)
function(add_simulation_test test_cpp)
# get exec name (remove extension from the abs path)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
target_link_libraries(${target_name} gtest gmock mg-utils mg-io mg-io-simulator mg-query-v2)
# sanitize
target_compile_options(${target_name} PRIVATE -fsanitize=${san})
target_link_options(${target_name} PRIVATE -fsanitize=${san})
target_link_libraries(${target_name} mg-storage-v3 mg-communication gtest gmock mg-utils mg-io mg-io-simulator Boost::headers mg-query-v2)
# register test
add_test(${target_name} ${exec_name})
add_dependencies(memgraph__simulation ${target_name})
endfunction(add_simulation_test)
add_simulation_test(basic_request.cpp address)
add_simulation_test(raft.cpp address)
add_simulation_test(trial_query_storage/query_storage_test.cpp address)
add_simulation_test(sharded_map.cpp address)
add_simulation_test(shard_request_manager.cpp address)
add_simulation_test(shard_rsm.cpp)

43
tests/simulation/common.hpp
View File

@ -40,6 +40,7 @@
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/value_conversions.hpp"
#include "utils/logging.hpp"
using memgraph::coordinator::Hlc;
@ -49,17 +50,17 @@ using memgraph::io::simulator::Simulator;
using memgraph::io::simulator::SimulatorConfig;
using memgraph::io::simulator::SimulatorStats;
using memgraph::io::simulator::SimulatorTransport;
using memgraph::msgs::CreateVerticesRequest;
using memgraph::msgs::CreateVerticesResponse;
using memgraph::msgs::ExpandOneRequest;
using memgraph::msgs::ExpandOneResponse;
using memgraph::msgs::ListedValues;
using memgraph::msgs::ScanVerticesRequest;
using memgraph::msgs::ScanVerticesResponse;
using memgraph::msgs::Value;
using memgraph::msgs::VertexId;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyValue;
using requests::CreateVerticesRequest;
using requests::CreateVerticesResponse;
using requests::ExpandOneRequest;
using requests::ExpandOneResponse;
using requests::ListedValues;
using requests::ScanVerticesRequest;
using requests::ScanVerticesResponse;
using requests::Value;
using requests::VertexId;
using ShardRsmKey = std::vector<memgraph::storage::v3::PropertyValue>;
@ -82,26 +83,28 @@ class MockedShardRsm {
// GetPropertiesResponse Read(GetPropertiesRequest rqst);
ScanVerticesResponse ReadImpl(ScanVerticesRequest rqst) {
ScanVerticesResponse ret;
if (!IsKeyInRange(rqst.start_id.primary_key)) {
auto as_prop_val = memgraph::storage::conversions::ConvertPropertyVector(rqst.start_id.second);
if (!IsKeyInRange(as_prop_val)) {
ret.success = false;
} else if (rqst.start_id.primary_key == ShardRsmKey{PropertyValue(0), PropertyValue(0)}) {
} else if (as_prop_val == ShardRsmKey{PropertyValue(0), PropertyValue(0)}) {
Value val(int64_t(0));
ret.next_start_id = std::make_optional<VertexId>();
ret.next_start_id->primary_key = ShardRsmKey{PropertyValue(1), PropertyValue(0)};
requests::ScanResultRow result;
result.props.insert(std::make_pair(requests::PropertyId::FromUint(0), val));
ret.next_start_id->second =
memgraph::storage::conversions::ConvertValueVector(ShardRsmKey{PropertyValue(1), PropertyValue(0)});
memgraph::msgs::ScanResultRow result;
result.props.push_back(std::make_pair(memgraph::msgs::PropertyId::FromUint(0), val));
ret.results.push_back(std::move(result));
ret.success = true;
} else if (rqst.start_id.primary_key == ShardRsmKey{PropertyValue(1), PropertyValue(0)}) {
requests::ScanResultRow result;
} else if (as_prop_val == ShardRsmKey{PropertyValue(1), PropertyValue(0)}) {
memgraph::msgs::ScanResultRow result;
Value val(int64_t(1));
result.props.insert(std::make_pair(requests::PropertyId::FromUint(0), val));
result.props.push_back(std::make_pair(memgraph::msgs::PropertyId::FromUint(0), val));
ret.results.push_back(std::move(result));
ret.success = true;
} else if (rqst.start_id.primary_key == ShardRsmKey{PropertyValue(12), PropertyValue(13)}) {
requests::ScanResultRow result;
} else if (as_prop_val == ShardRsmKey{PropertyValue(12), PropertyValue(13)}) {
memgraph::msgs::ScanResultRow result;
Value val(int64_t(444));
result.props.insert(std::make_pair(requests::PropertyId::FromUint(0), val));
result.props.push_back(std::make_pair(memgraph::msgs::PropertyId::FromUint(0), val));
ret.results.push_back(std::move(result));
ret.success = true;
} else {

45
tests/simulation/shard_request_manager.cpp
View File

@ -66,15 +66,15 @@ using memgraph::io::simulator::Simulator;
using memgraph::io::simulator::SimulatorConfig;
using memgraph::io::simulator::SimulatorStats;
using memgraph::io::simulator::SimulatorTransport;
using memgraph::msgs::CreateVerticesRequest;
using memgraph::msgs::CreateVerticesResponse;
using memgraph::msgs::ListedValues;
using memgraph::msgs::NewVertexLabel;
using memgraph::msgs::ScanVerticesRequest;
using memgraph::msgs::ScanVerticesResponse;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::SchemaProperty;
using memgraph::utils::BasicResult;
using requests::CreateVerticesRequest;
using requests::CreateVerticesResponse;
using requests::ListedValues;
using requests::NewVertexLabel;
using requests::ScanVerticesRequest;
using requests::ScanVerticesResponse;
namespace {
@ -150,21 +150,21 @@ void RunStorageRaft(Raft<IoImpl, MockedShardRsm, WriteRequests, WriteResponses,
template <typename ShardRequestManager>
void TestScanAll(ShardRequestManager &io) {
requests::ExecutionState<ScanVerticesRequest> state{.label = "test_label"};
memgraph::msgs::ExecutionState<ScanVerticesRequest> state{.label = "test_label"};
auto result = io.Request(state);
MG_ASSERT(result.size() == 2);
{
auto prop = result[0].GetProperty(requests::PropertyId::FromUint(0));
auto prop = result[0].GetProperty(memgraph::msgs::PropertyId::FromUint(0));
MG_ASSERT(prop.int_v == 0);
prop = result[1].GetProperty(requests::PropertyId::FromUint(0));
prop = result[1].GetProperty(memgraph::msgs::PropertyId::FromUint(0));
MG_ASSERT(prop.int_v == 444);
}
result = io.Request(state);
{
MG_ASSERT(result.size() == 1);
auto prop = result[0].GetProperty(requests::PropertyId::FromUint(0));
auto prop = result[0].GetProperty(memgraph::msgs::PropertyId::FromUint(0));
MG_ASSERT(prop.int_v == 1);
}
@ -175,12 +175,14 @@ void TestScanAll(ShardRequestManager &io) {
template <typename ShardRequestManager>
void TestCreateVertices(ShardRequestManager &io) {
using PropVal = memgraph::storage::v3::PropertyValue;
requests::ExecutionState<CreateVerticesRequest> state;
std::vector<requests::NewVertex> new_vertices;
using PropVal = memgraph::msgs::Value;
memgraph::msgs::ExecutionState<CreateVerticesRequest> state;
std::vector<memgraph::msgs::NewVertex> new_vertices;
auto label_id = io.LabelNameToLabelId("test_label");
requests::NewVertex a1{.label_ids = label_id, .primary_key = {PropVal(1), PropVal(0)}};
requests::NewVertex a2{.label_ids = label_id, .primary_key = {PropVal(13), PropVal(13)}};
memgraph::msgs::NewVertex a1{.primary_key = {PropVal(int64_t(1)), PropVal(int64_t(0))}};
a1.label_ids.push_back({label_id});
memgraph::msgs::NewVertex a2{.primary_key = {PropVal(int64_t(13)), PropVal(int64_t(13))}};
a2.label_ids.push_back({label_id});
new_vertices.push_back(std::move(a1));
new_vertices.push_back(std::move(a2));
@ -205,17 +207,25 @@ int main() {
};
auto simulator = Simulator(config);
const auto one_second = std::chrono::seconds(1);
Io<SimulatorTransport> cli_io = simulator.RegisterNew();
cli_io.SetDefaultTimeout(one_second);
// Register
Io<SimulatorTransport> a_io_1 = simulator.RegisterNew();
a_io_1.SetDefaultTimeout(one_second);
Io<SimulatorTransport> a_io_2 = simulator.RegisterNew();
a_io_2.SetDefaultTimeout(one_second);
Io<SimulatorTransport> a_io_3 = simulator.RegisterNew();
a_io_3.SetDefaultTimeout(one_second);
Io<SimulatorTransport> b_io_1 = simulator.RegisterNew();
b_io_1.SetDefaultTimeout(one_second);
Io<SimulatorTransport> b_io_2 = simulator.RegisterNew();
b_io_2.SetDefaultTimeout(one_second);
Io<SimulatorTransport> b_io_3 = simulator.RegisterNew();
b_io_3.SetDefaultTimeout(one_second);
// Preconfigure coordinator with kv shard 'A' and 'B'
auto sm1 = CreateDummyShardmap(a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress(), b_io_1.GetAddress(),
@ -268,8 +278,11 @@ int main() {
// Spin up coordinators
Io<SimulatorTransport> c_io_1 = simulator.RegisterNew();
c_io_1.SetDefaultTimeout(one_second);
Io<SimulatorTransport> c_io_2 = simulator.RegisterNew();
c_io_2.SetDefaultTimeout(one_second);
Io<SimulatorTransport> c_io_3 = simulator.RegisterNew();
c_io_3.SetDefaultTimeout(one_second);
std::vector<Address> c_addrs = {c_io_1.GetAddress(), c_io_2.GetAddress(), c_io_3.GetAddress()};
@ -296,7 +309,7 @@ int main() {
// also get the current shard map
CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, c_addrs[0], c_addrs);
requests::ShardRequestManager<SimulatorTransport> io(std::move(coordinator_client), std::move(cli_io));
memgraph::msgs::ShardRequestManager<SimulatorTransport> io(std::move(coordinator_client), std::move(cli_io));
io.StartTransaction();
TestScanAll(io);

366
tests/simulation/shard_rsm.cpp Normal file
View File

@ -0,0 +1,366 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <iostream>
#include <optional>
#include <thread>
#include <utility>
#include <vector>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/rsm/raft.hpp"
#include "io/rsm/rsm_client.hpp"
#include "io/simulator/simulator.hpp"
#include "io/simulator/simulator_transport.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/shard_rsm.hpp"
#include "storage/v3/view.hpp"
#include "utils/result.hpp"
namespace memgraph::storage::v3::tests {
using io::Address;
using io::Io;
using io::ResponseEnvelope;
using io::ResponseFuture;
using io::Time;
using io::TimedOut;
using io::rsm::Raft;
using io::rsm::ReadRequest;
using io::rsm::ReadResponse;
using io::rsm::RsmClient;
using io::rsm::WriteRequest;
using io::rsm::WriteResponse;
using io::simulator::Simulator;
using io::simulator::SimulatorConfig;
using io::simulator::SimulatorStats;
using io::simulator::SimulatorTransport;
using utils::BasicResult;
using msgs::ReadRequests;
using msgs::ReadResponses;
using msgs::WriteRequests;
using msgs::WriteResponses;
using ShardClient = RsmClient<SimulatorTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
using ConcreteShardRsm = Raft<SimulatorTransport, ShardRsm, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
// TODO(gvolfing) test vertex deletion with DETACH_DELETE as well
template <typename IoImpl>
void RunShardRaft(Raft<IoImpl, ShardRsm, WriteRequests, WriteResponses, ReadRequests, ReadResponses> server) {
server.Run();
}
namespace {
uint64_t GetTransactionId() {
static uint64_t transaction_id = 0;
return transaction_id++;
}
uint64_t GetUniqueInteger() {
static uint64_t prop_val_val = 1001;
return prop_val_val++;
}
LabelId get_primary_label() { return LabelId::FromUint(0); }
SchemaProperty get_schema_property() {
return {.property_id = PropertyId::FromUint(0), .type = common::SchemaType::INT};
}
msgs::PrimaryKey GetPrimaryKey(int64_t value) {
msgs::Value prop_val(static_cast<int64_t>(value));
msgs::PrimaryKey primary_key = {prop_val};
return primary_key;
}
msgs::NewVertex get_new_vertex(int64_t value) {
// Specify Labels.
msgs::Label label1 = {.id = LabelId::FromUint(1)};
std::vector<msgs::Label> label_ids = {label1};
// Specify primary key.
msgs::PrimaryKey primary_key = GetPrimaryKey(value);
// Specify properties
auto val1 = msgs::Value(static_cast<int64_t>(value));
auto prop1 = std::make_pair(PropertyId::FromUint(0), val1);
auto val2 = msgs::Value(static_cast<int64_t>(value));
auto prop2 = std::make_pair(PropertyId::FromUint(1), val1);
std::vector<std::pair<PropertyId, msgs::Value>> properties{prop1, prop2};
// NewVertex
return {.label_ids = label_ids, .primary_key = primary_key, .properties = properties};
}
// TODO(gvolfing) maybe rename that something that makes sense.
std::vector<std::vector<msgs::Value>> GetValuePrimaryKeysWithValue(int64_t value) {
msgs::Value val(static_cast<int64_t>(value));
return {{val}};
}
} // namespace
// attempts to sending different requests
namespace {
bool AttemtpToCreateVertex(ShardClient &client, int64_t value) {
msgs::NewVertex vertex = get_new_vertex(value);
auto create_req = msgs::CreateVerticesRequest{};
create_req.new_vertices = {vertex};
create_req.transaction_id.logical_id = GetTransactionId();
while (true) {
auto write_res = client.SendWriteRequest(create_req);
if (write_res.HasError()) {
continue;
}
auto write_response_result = write_res.GetValue();
auto write_response = std::get<msgs::CreateVerticesResponse>(write_response_result);
return write_response.success;
}
}
bool AttemptToAddEdge(ShardClient &client, int64_t value_of_vertex_1, int64_t value_of_vertex_2, int64_t edge_gid,
int64_t edge_type_id) {
auto id = msgs::EdgeId{};
msgs::Label label = {.id = get_primary_label()};
auto src = std::make_pair(label, GetPrimaryKey(value_of_vertex_1));
auto dst = std::make_pair(label, GetPrimaryKey(value_of_vertex_2));
id.gid = edge_gid;
auto type = msgs::EdgeType{};
type.id = edge_type_id;
auto edge = msgs::Edge{};
edge.id = id;
edge.type = type;
edge.src = src;
edge.dst = dst;
msgs::CreateEdgesRequest create_req{};
create_req.edges = {edge};
create_req.transaction_id.logical_id = GetTransactionId();
while (true) {
auto write_res = client.SendWriteRequest(create_req);
if (write_res.HasError()) {
continue;
}
auto write_response_result = write_res.GetValue();
auto write_response = std::get<msgs::CreateEdgesResponse>(write_response_result);
return write_response.success;
}
}
std::tuple<size_t, std::optional<msgs::VertexId>> AttemptToScanAllWithBatchLimit(ShardClient &client,
msgs::VertexId start_id,
uint64_t batch_limit) {
msgs::ScanVerticesRequest scan_req{};
scan_req.batch_limit = batch_limit;
scan_req.filter_expressions = std::nullopt;
scan_req.props_to_return = std::nullopt;
scan_req.start_id = start_id;
scan_req.storage_view = msgs::StorageView::OLD;
scan_req.transaction_id.logical_id = GetTransactionId();
while (true) {
auto read_res = client.SendReadRequest(scan_req);
if (read_res.HasError()) {
continue;
}
auto write_response_result = read_res.GetValue();
auto write_response = std::get<msgs::ScanVerticesResponse>(write_response_result);
MG_ASSERT(write_response.success);
return {write_response.results.size(), write_response.next_start_id};
}
}
} // namespace
// tests
namespace {
void TestCreateVertices(ShardClient &client) { MG_ASSERT(AttemtpToCreateVertex(client, GetUniqueInteger())); }
void TestAddEdge(ShardClient &client) {
auto unique_prop_val_1 = GetUniqueInteger();
auto unique_prop_val_2 = GetUniqueInteger();
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_1));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_2));
auto edge_gid = GetUniqueInteger();
auto edge_type_id = GetUniqueInteger();
MG_ASSERT(AttemptToAddEdge(client, unique_prop_val_1, unique_prop_val_2, edge_gid, edge_type_id));
}
void TestScanAllOneGo(ShardClient &client) {
auto unique_prop_val_1 = GetUniqueInteger();
auto unique_prop_val_2 = GetUniqueInteger();
auto unique_prop_val_3 = GetUniqueInteger();
auto unique_prop_val_4 = GetUniqueInteger();
auto unique_prop_val_5 = GetUniqueInteger();
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_1));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_2));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_3));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_4));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_5));
msgs::Label prim_label = {.id = get_primary_label()};
msgs::PrimaryKey prim_key = {msgs::Value(static_cast<int64_t>(unique_prop_val_1))};
msgs::VertexId v_id = {prim_label, prim_key};
auto [result_size, next_id] = AttemptToScanAllWithBatchLimit(client, v_id, 5);
MG_ASSERT(result_size == 5);
}
void TestScanAllWithSmallBatchSize(ShardClient &client) {
auto unique_prop_val_1 = GetUniqueInteger();
auto unique_prop_val_2 = GetUniqueInteger();
auto unique_prop_val_3 = GetUniqueInteger();
auto unique_prop_val_4 = GetUniqueInteger();
auto unique_prop_val_5 = GetUniqueInteger();
auto unique_prop_val_6 = GetUniqueInteger();
auto unique_prop_val_7 = GetUniqueInteger();
auto unique_prop_val_8 = GetUniqueInteger();
auto unique_prop_val_9 = GetUniqueInteger();
auto unique_prop_val_10 = GetUniqueInteger();
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_1));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_2));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_3));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_4));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_5));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_6));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_7));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_8));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_9));
MG_ASSERT(AttemtpToCreateVertex(client, unique_prop_val_10));
msgs::Label prim_label = {.id = get_primary_label()};
msgs::PrimaryKey prim_key1 = {msgs::Value(static_cast<int64_t>(unique_prop_val_1))};
msgs::VertexId v_id_1 = {prim_label, prim_key1};
auto [result_size1, next_id1] = AttemptToScanAllWithBatchLimit(client, v_id_1, 3);
MG_ASSERT(result_size1 == 3);
auto [result_size2, next_id2] = AttemptToScanAllWithBatchLimit(client, next_id1.value(), 3);
MG_ASSERT(result_size2 == 3);
auto [result_size3, next_id3] = AttemptToScanAllWithBatchLimit(client, next_id2.value(), 3);
MG_ASSERT(result_size3 == 3);
auto [result_size4, next_id4] = AttemptToScanAllWithBatchLimit(client, next_id3.value(), 3);
MG_ASSERT(result_size4 == 1);
MG_ASSERT(!next_id4);
}
} // namespace
int TestMessages() {
SimulatorConfig config{
.drop_percent = 0,
.perform_timeouts = false,
.scramble_messages = false,
.rng_seed = 0,
.start_time = Time::min() + std::chrono::microseconds{256 * 1024},
.abort_time = Time::min() + std::chrono::microseconds{4 * 8 * 1024 * 1024},
};
auto simulator = Simulator(config);
Io<SimulatorTransport> shard_server_io_1 = simulator.RegisterNew();
const auto shard_server_1_address = shard_server_io_1.GetAddress();
Io<SimulatorTransport> shard_server_io_2 = simulator.RegisterNew();
const auto shard_server_2_address = shard_server_io_2.GetAddress();
Io<SimulatorTransport> shard_server_io_3 = simulator.RegisterNew();
const auto shard_server_3_address = shard_server_io_3.GetAddress();
Io<SimulatorTransport> shard_client_io = simulator.RegisterNew();
PropertyValue min_pk(static_cast<int64_t>(0));
std::vector<PropertyValue> min_prim_key = {min_pk};
PropertyValue max_pk(static_cast<int64_t>(10000000));
std::vector<PropertyValue> max_prim_key = {max_pk};
auto shard_ptr1 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key);
auto shard_ptr2 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key);
auto shard_ptr3 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key);
shard_ptr1->CreateSchema(get_primary_label(), {get_schema_property()});
shard_ptr2->CreateSchema(get_primary_label(), {get_schema_property()});
shard_ptr3->CreateSchema(get_primary_label(), {get_schema_property()});
std::vector<Address> address_for_1{shard_server_2_address, shard_server_3_address};
std::vector<Address> address_for_2{shard_server_1_address, shard_server_3_address};
std::vector<Address> address_for_3{shard_server_1_address, shard_server_2_address};
ConcreteShardRsm shard_server1(std::move(shard_server_io_1), address_for_1, ShardRsm(std::move(shard_ptr1)));
ConcreteShardRsm shard_server2(std::move(shard_server_io_2), address_for_2, ShardRsm(std::move(shard_ptr2)));
ConcreteShardRsm shard_server3(std::move(shard_server_io_3), address_for_3, ShardRsm(std::move(shard_ptr3)));
auto server_thread1 = std::jthread([&shard_server1]() { shard_server1.Run(); });
auto server_thread2 = std::jthread([&shard_server2]() { shard_server2.Run(); });
auto server_thread3 = std::jthread([&shard_server3]() { shard_server3.Run(); });
simulator.IncrementServerCountAndWaitForQuiescentState(shard_server_1_address);
simulator.IncrementServerCountAndWaitForQuiescentState(shard_server_2_address);
simulator.IncrementServerCountAndWaitForQuiescentState(shard_server_3_address);
std::cout << "Beginning test after servers have become quiescent." << std::endl;
std::vector server_addrs = {shard_server_1_address, shard_server_2_address, shard_server_3_address};
ShardClient client(shard_client_io, shard_server_1_address, server_addrs);
TestCreateVertices(client);
TestAddEdge(client);
TestScanAllOneGo(client);
TestScanAllWithSmallBatchSize(client);
simulator.ShutDown();
SimulatorStats stats = simulator.Stats();
std::cout << "total messages: " << stats.total_messages << std::endl;
std::cout << "dropped messages: " << stats.dropped_messages << std::endl;
std::cout << "timed out requests: " << stats.timed_out_requests << std::endl;
std::cout << "total requests: " << stats.total_requests << std::endl;
std::cout << "total responses: " << stats.total_responses << std::endl;
std::cout << "simulator ticks: " << stats.simulator_ticks << std::endl;
std::cout << "========================== SUCCESS :) ==========================" << std::endl;
return 0;
}
} // namespace memgraph::storage::v3::tests
int main() { return memgraph::storage::v3::tests::TestMessages(); }

25
tests/unit/CMakeLists.txt
View File

@ -319,13 +319,16 @@ add_unit_test(storage_v2_isolation_level.cpp)
target_link_libraries(${test_prefix}storage_v2_isolation_level mg-storage-v2)
# Test mg-storage-v3
add_library(storage_v3_test_utils storage_v3_test_utils.cpp)
target_link_libraries(storage_v3_test_utils mg-storage-v3)
add_unit_test(storage_v3.cpp)
target_link_libraries(${test_prefix}storage_v3 mg-storage-v3)
target_link_libraries(${test_prefix}storage_v3 mg-storage-v3 storage_v3_test_utils)
add_unit_test(storage_v3_schema.cpp)
target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3)
# Test mg-query-v3
# Test mg-query-v2
# These are commented out because of the new TypedValue in the query engine
#add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp)
#target_link_libraries(${test_prefix}query_v2_interpreter mg-storage-v3 mg-query-v2 mg-communication)
@ -360,6 +363,24 @@ target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3)
add_unit_test(query_v2_dummy_test.cpp)
target_link_libraries(${test_prefix}query_v2_dummy_test mg-query-v2)
add_unit_test(storage_v3_property_store.cpp)
target_link_libraries(${test_prefix}storage_v3_property_store mg-storage-v3 fmt)
add_unit_test(storage_v3_key_store.cpp)
target_link_libraries(${test_prefix}storage_v3_key_store mg-storage-v3 rapidcheck rapidcheck_gtest)
add_unit_test(storage_v3_indices.cpp)
target_link_libraries(${test_prefix}storage_v3_indices mg-storage-v3)
add_unit_test(storage_v3_vertex_accessors.cpp)
target_link_libraries(${test_prefix}storage_v3_vertex_accessors mg-storage-v3)
add_unit_test(storage_v3_edge.cpp)
target_link_libraries(${test_prefix}storage_v3_edge mg-storage-v3)
add_unit_test(replication_persistence_helper.cpp)
target_link_libraries(${test_prefix}replication_persistence_helper mg-storage-v2)
# Test mg-auth
if(MG_ENTERPRISE)
add_unit_test(auth.cpp)

3054
tests/unit/query_v2_interpreter.cpp
View File

File diff suppressed because it is too large Load Diff

8
tests/unit/query_v2_query_plan_v2_create_set_remove_delete.cpp
View File

@ -94,7 +94,7 @@ TEST_F(QueryPlanCRUDTest, ScanAll) {
auto dba = db.Access();
for (int i = 0; i < 42; ++i) {
auto v = *dba.CreateVertexAndValidate(label, {}, {{property, storage::v3::PropertyValue(i)}});
ASSERT_TRUE(v.SetProperty(property, storage::v3::PropertyValue(i)).HasValue());
ASSERT_TRUE(v.SetPropertyAndValidate(property, storage::v3::PropertyValue(i)).HasValue());
}
EXPECT_FALSE(dba.Commit().HasError());
}
@ -120,13 +120,13 @@ TEST_F(QueryPlanCRUDTest, ScanAllByLabel) {
// Add some unlabeled vertices
for (int i = 0; i < 12; ++i) {
auto v = *dba.CreateVertexAndValidate(label, {}, {{property, storage::v3::PropertyValue(i)}});
ASSERT_TRUE(v.SetProperty(property, storage::v3::PropertyValue(i)).HasValue());
ASSERT_TRUE(v.SetPropertyAndValidate(property, storage::v3::PropertyValue(i)).HasValue());
}
// Add labeled vertices
for (int i = 0; i < 42; ++i) {
auto v = *dba.CreateVertexAndValidate(label, {}, {{property, storage::v3::PropertyValue(i)}});
ASSERT_TRUE(v.SetProperty(property, storage::v3::PropertyValue(i)).HasValue());
ASSERT_TRUE(v.AddLabel(label2).HasValue());
ASSERT_TRUE(v.SetPropertyAndValidate(property, storage::v3::PropertyValue(i)).HasValue());
ASSERT_TRUE(v.AddLabelAndValidate(label2).HasValue());
}
EXPECT_FALSE(dba.Commit().HasError());
}

4
tests/unit/result_stream_faker.hpp
View File

@ -27,7 +27,7 @@
*/
class ResultStreamFaker {
public:
explicit ResultStreamFaker(memgraph::storage::v3::Storage *store) : store_(store) {}
explicit ResultStreamFaker(memgraph::storage::v3::Shard *store) : store_(store) {}
ResultStreamFaker(const ResultStreamFaker &) = delete;
ResultStreamFaker &operator=(const ResultStreamFaker &) = delete;
@ -125,7 +125,7 @@ class ResultStreamFaker {
}
private:
memgraph::storage::v3::Storage *store_;
memgraph::storage::v3::Shard *store_;
// the data that the record stream can accept
std::vector<std::string> header_;
std::vector<std::vector<memgraph::communication::bolt::Value>> results_;

2614
tests/unit/storage_v3.cpp
View File

File diff suppressed because it is too large Load Diff

5274
tests/unit/storage_v3_edge.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

989
tests/unit/storage_v3_indices.cpp Normal file
View File

@ -0,0 +1,989 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <gmock/gmock-matchers.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <cstdint>
#include "storage/v3/id_types.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/storage.hpp"
#include "storage/v3/temporal.hpp"
// NOLINTNEXTLINE(google-build-using-namespace)
using testing::IsEmpty;
using testing::Pair;
using testing::UnorderedElementsAre;
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define ASSERT_NO_ERROR(result) ASSERT_FALSE((result).HasError())
namespace memgraph::storage::v3::tests {
class IndexTest : public testing::Test {
protected:
void SetUp() override {
ASSERT_TRUE(
storage.CreateSchema(primary_label, {storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}}));
}
NameIdMapper id_mapper;
const std::vector<PropertyValue> pk{PropertyValue{0}};
const LabelId primary_label{NameToLabelId("label")};
Shard storage{primary_label, pk, std::nullopt};
const PropertyId primary_property{NameToPropertyId("property")};
const PropertyId prop_id{NameToPropertyId("id")};
const PropertyId prop_val{NameToPropertyId("val")};
const LabelId label1{NameToLabelId("label1")};
const LabelId label2{NameToLabelId("label2")};
int primary_key_id{0};
int vertex_id{0};
LabelId NameToLabelId(std::string_view label_name) { return LabelId::FromUint(id_mapper.NameToId(label_name)); }
PropertyId NameToPropertyId(std::string_view property_name) {
return PropertyId::FromUint(id_mapper.NameToId(property_name));
}
VertexAccessor CreateVertex(Shard::Accessor *accessor) {
auto vertex = *accessor->CreateVertexAndValidate(
primary_label, {},
{{primary_property, PropertyValue(primary_key_id++)}, {prop_id, PropertyValue(vertex_id++)}});
return vertex;
}
template <class TIterable>
std::vector<int64_t> GetIds(TIterable iterable, View view = View::OLD) {
std::vector<int64_t> ret;
for (auto vertex : iterable) {
ret.push_back(vertex.GetProperty(prop_id, view)->ValueInt());
}
return ret;
}
template <class TIterable>
std::vector<int64_t> GetPrimaryKeyIds(TIterable iterable, View view = View::OLD) {
std::vector<int64_t> ret;
for (auto vertex : iterable) {
EXPECT_TRUE(vertex.PrimaryKey(view).HasValue());
const auto pk = vertex.PrimaryKey(view).GetValue();
EXPECT_EQ(pk.size(), 1);
ret.push_back(pk[0].ValueInt());
}
return ret;
}
};
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexCreate) {
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelIndexExists(label1));
}
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
{
auto acc = storage.Access();
for (int i = 0; i < 10; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
ASSERT_NO_ERROR(acc.Commit());
}
EXPECT_TRUE(storage.CreateIndex(label1));
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
}
{
auto acc = storage.Access();
for (int i = 10; i < 20; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
acc.Abort();
}
{
auto acc = storage.Access();
for (int i = 10; i < 20; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
ASSERT_NO_ERROR(acc.Commit());
}
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 21, 23, 25, 27, 29));
ASSERT_NO_ERROR(acc.Commit());
}
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexDrop) {
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelIndexExists(label1));
}
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
{
auto acc = storage.Access();
for (int i = 0; i < 10; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
ASSERT_NO_ERROR(acc.Commit());
}
EXPECT_TRUE(storage.CreateIndex(label1));
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
}
EXPECT_TRUE(storage.DropIndex(label1));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelIndexExists(label1));
}
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
EXPECT_FALSE(storage.DropIndex(label1));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelIndexExists(label1));
}
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
{
auto acc = storage.Access();
for (int i = 10; i < 20; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
ASSERT_NO_ERROR(acc.Commit());
}
EXPECT_TRUE(storage.CreateIndex(label1));
{
auto acc = storage.Access();
EXPECT_TRUE(acc.LabelIndexExists(label1));
}
EXPECT_THAT(storage.ListAllIndices().label, UnorderedElementsAre(label1));
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(1, 3, 5, 7, 9, 11, 13, 15, 17, 19));
}
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexBasic) {
// The following steps are performed and index correctness is validated after
// each step:
// 1. Create 10 vertices numbered from 0 to 9.
// 2. Add Label1 to odd numbered, and Label2 to even numbered vertices.
// 3. Remove Label1 from odd numbered vertices, and add it to even numbered
// vertices.
// 4. Delete even numbered vertices.
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_TRUE(storage.CreateIndex(label2));
auto acc = storage.Access();
EXPECT_THAT(storage.ListAllIndices().label, UnorderedElementsAre(label1, label2));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), IsEmpty());
for (int i = 0; i < 10; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
for (auto vertex : acc.Vertices(View::OLD)) {
int64_t id = vertex.GetProperty(prop_id, View::OLD)->ValueInt();
if (id % 2) {
ASSERT_NO_ERROR(vertex.RemoveLabelAndValidate(label1));
} else {
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
}
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
for (auto vertex : acc.Vertices(View::OLD)) {
int64_t id = vertex.GetProperty(prop_id, View::OLD)->ValueInt();
if (id % 2 == 0) {
ASSERT_NO_ERROR(acc.DeleteVertex(&vertex));
}
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), IsEmpty());
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, View::NEW), View::NEW), IsEmpty());
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexDuplicateVersions) {
// By removing labels and adding them again we create duplicate entries for
// the same vertex in the index (they only differ by the timestamp). This test
// checks that duplicates are properly filtered out.
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_TRUE(storage.CreateIndex(label2));
{
auto acc = storage.Access();
for (int i = 0; i < 5; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
ASSERT_NO_ERROR(acc.Commit());
}
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
for (auto vertex : acc.Vertices(View::OLD)) {
ASSERT_NO_ERROR(vertex.RemoveLabelAndValidate(label1));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), IsEmpty());
for (auto vertex : acc.Vertices(View::OLD)) {
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexTransactionalIsolation) {
// Check that transactions only see entries they are supposed to see.
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_TRUE(storage.CreateIndex(label2));
auto acc_before = storage.Access();
auto acc = storage.Access();
auto acc_after = storage.Access();
for (int i = 0; i < 5; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc_before.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after.Vertices(label1, View::NEW), View::NEW), IsEmpty());
ASSERT_NO_ERROR(acc.Commit());
auto acc_after_commit = storage.Access();
EXPECT_THAT(GetIds(acc_before.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after.Vertices(label1, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after_commit.Vertices(label1, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelIndexCountEstimate) {
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_TRUE(storage.CreateIndex(label2));
auto acc = storage.Access();
for (int i = 0; i < 20; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 3 ? label1 : label2));
}
EXPECT_EQ(acc.ApproximateVertexCount(label1), 13);
EXPECT_EQ(acc.ApproximateVertexCount(label2), 7);
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexCreateAndDrop) {
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
EXPECT_TRUE(storage.CreateIndex(label1, prop_id));
{
auto acc = storage.Access();
EXPECT_TRUE(acc.LabelPropertyIndexExists(label1, prop_id));
}
EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(std::make_pair(label1, prop_id)));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelPropertyIndexExists(label2, prop_id));
}
EXPECT_FALSE(storage.CreateIndex(label1, prop_id));
EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(std::make_pair(label1, prop_id)));
EXPECT_TRUE(storage.CreateIndex(label2, prop_id));
{
auto acc = storage.Access();
EXPECT_TRUE(acc.LabelPropertyIndexExists(label2, prop_id));
}
EXPECT_THAT(storage.ListAllIndices().label_property,
UnorderedElementsAre(std::make_pair(label1, prop_id), std::make_pair(label2, prop_id)));
EXPECT_TRUE(storage.DropIndex(label1, prop_id));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelPropertyIndexExists(label1, prop_id));
}
EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(std::make_pair(label2, prop_id)));
EXPECT_FALSE(storage.DropIndex(label1, prop_id));
EXPECT_TRUE(storage.DropIndex(label2, prop_id));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelPropertyIndexExists(label2, prop_id));
}
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
}
// The following three tests are almost an exact copy-paste of the corresponding
// label index tests. We request all vertices with given label and property from
// the index, without range filtering. Range filtering is tested in a separate
// test.
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexBasic) {
storage.CreateIndex(label1, prop_val);
storage.CreateIndex(label2, prop_val);
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty());
for (int i = 0; i < 10; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(i % 2 ? label1 : label2));
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i)));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
for (auto vertex : acc.Vertices(View::OLD)) {
int64_t id = vertex.GetProperty(prop_id, View::OLD)->ValueInt();
if (id % 2) {
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue()));
} else {
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
}
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 2, 4, 6, 8));
for (auto vertex : acc.Vertices(View::OLD)) {
int64_t id = vertex.GetProperty(prop_id, View::OLD)->ValueInt();
if (id % 2 == 0) {
ASSERT_NO_ERROR(acc.DeleteVertex(&vertex));
}
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(1, 3, 5, 7, 9));
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 2, 4, 6, 8));
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::NEW), View::NEW), IsEmpty());
acc.AdvanceCommand();
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::OLD), View::OLD), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc.Vertices(label2, prop_val, View::NEW), View::NEW), IsEmpty());
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexDuplicateVersions) {
storage.CreateIndex(label1, prop_val);
{
auto acc = storage.Access();
for (int i = 0; i < 5; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i)));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
ASSERT_NO_ERROR(acc.Commit());
}
{
auto acc = storage.Access();
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
for (auto vertex : acc.Vertices(View::OLD)) {
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue()));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
for (auto vertex : acc.Vertices(View::OLD)) {
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(42)));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexTransactionalIsolation) {
storage.CreateIndex(label1, prop_val);
auto acc_before = storage.Access();
auto acc = storage.Access();
auto acc_after = storage.Access();
for (int i = 0; i < 5; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i)));
}
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, View::NEW), View::NEW), UnorderedElementsAre(0, 1, 2, 3, 4));
EXPECT_THAT(GetIds(acc_before.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
ASSERT_NO_ERROR(acc.Commit());
auto acc_after_commit = storage.Access();
EXPECT_THAT(GetIds(acc_before.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after.Vertices(label1, prop_val, View::NEW), View::NEW), IsEmpty());
EXPECT_THAT(GetIds(acc_after_commit.Vertices(label1, prop_val, View::NEW), View::NEW),
UnorderedElementsAre(0, 1, 2, 3, 4));
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexFiltering) {
// We insert vertices with values:
// 0 0.0 1 1.0 2 2.0 3 3.0 4 4.0
// Then we check all combinations of inclusive and exclusive bounds.
// We also have a mix of doubles and integers to verify that they are sorted
// properly.
storage.CreateIndex(label1, prop_val);
{
auto acc = storage.Access();
for (int i = 0; i < 10; ++i) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, i % 2 ? PropertyValue(i / 2) : PropertyValue(i / 2.0)));
}
ASSERT_NO_ERROR(acc.Commit());
}
{
auto acc = storage.Access();
for (int i = 0; i < 5; ++i) {
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, PropertyValue(i), View::OLD)),
UnorderedElementsAre(2 * i, 2 * i + 1));
}
// [1, +inf>
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundInclusive(PropertyValue(1)),
std::nullopt, View::OLD)),
UnorderedElementsAre(2, 3, 4, 5, 6, 7, 8, 9));
// <1, +inf>
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundExclusive(PropertyValue(1)),
std::nullopt, View::OLD)),
UnorderedElementsAre(4, 5, 6, 7, 8, 9));
// <-inf, 3]
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, std::nullopt,
memgraph::utils::MakeBoundInclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(0, 1, 2, 3, 4, 5, 6, 7));
// <-inf, 3>
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, std::nullopt,
memgraph::utils::MakeBoundExclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(0, 1, 2, 3, 4, 5));
// [1, 3]
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundInclusive(PropertyValue(1)),
memgraph::utils::MakeBoundInclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(2, 3, 4, 5, 6, 7));
// <1, 3]
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundExclusive(PropertyValue(1)),
memgraph::utils::MakeBoundInclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(4, 5, 6, 7));
// [1, 3>
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundInclusive(PropertyValue(1)),
memgraph::utils::MakeBoundExclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(2, 3, 4, 5));
// <1, 3>
EXPECT_THAT(GetIds(acc.Vertices(label1, prop_val, memgraph::utils::MakeBoundExclusive(PropertyValue(1)),
memgraph::utils::MakeBoundExclusive(PropertyValue(3)), View::OLD)),
UnorderedElementsAre(4, 5));
}
}
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(IndexTest, LabelPropertyIndexCountEstimate) {
storage.CreateIndex(label1, prop_val);
auto acc = storage.Access();
for (int i = 1; i <= 10; ++i) {
for (int j = 0; j < i; ++j) {
auto vertex = CreateVertex(&acc);
ASSERT_NO_ERROR(vertex.AddLabelAndValidate(label1));
ASSERT_NO_ERROR(vertex.SetPropertyAndValidate(prop_val, PropertyValue(i)));
}
}
EXPECT_EQ(acc.ApproximateVertexCount(label1, prop_val), 55);
for (int i = 1; i <= 10; ++i) {
EXPECT_EQ(acc.ApproximateVertexCount(label1, prop_val, PropertyValue(i)), i);
}
EXPECT_EQ(acc.ApproximateVertexCount(label1, prop_val, memgraph::utils::MakeBoundInclusive(PropertyValue(2)),
memgraph::utils::MakeBoundInclusive(PropertyValue(6))),
2 + 3 + 4 + 5 + 6);
}
TEST_F(IndexTest, LabelPropertyIndexMixedIteration) {
storage.CreateIndex(label1, prop_val);
const std::array temporals{TemporalData{TemporalType::Date, 23}, TemporalData{TemporalType::Date, 28},
TemporalData{TemporalType::LocalDateTime, 20}};
std::vector<PropertyValue> values = {
PropertyValue(false),
PropertyValue(true),
PropertyValue(-std::numeric_limits<double>::infinity()),
PropertyValue(std::numeric_limits<int64_t>::min()),
PropertyValue(-1),
PropertyValue(-0.5),
PropertyValue(0),
PropertyValue(0.5),
PropertyValue(1),
PropertyValue(1.5),
PropertyValue(2),
PropertyValue(std::numeric_limits<int64_t>::max()),
PropertyValue(std::numeric_limits<double>::infinity()),
PropertyValue(""),
PropertyValue("a"),
PropertyValue("b"),
PropertyValue("c"),
PropertyValue(std::vector<PropertyValue>()),
PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(2)}),
PropertyValue(std::map<std::string, PropertyValue>()),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5)}}),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}}),
PropertyValue(temporals[0]),
PropertyValue(temporals[1]),
PropertyValue(temporals[2]),
};
// Create vertices, each with one of the values above.
{
auto acc = storage.Access();
for (const auto &value : values) {
auto v = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(primary_key_id++)}});
ASSERT_TRUE(v->AddLabelAndValidate(label1).HasValue());
ASSERT_TRUE(v->SetPropertyAndValidate(prop_val, value).HasValue());
}
ASSERT_FALSE(acc.Commit().HasError());
}
// Verify that all nodes are in the index.
{
auto acc = storage.Access();
auto iterable = acc.Vertices(label1, prop_val, View::OLD);
auto it = iterable.begin();
for (const auto &value : values) {
ASSERT_NE(it, iterable.end());
auto vertex = *it;
auto maybe_value = vertex.GetProperty(prop_val, View::OLD);
ASSERT_TRUE(maybe_value.HasValue());
ASSERT_EQ(value, *maybe_value);
++it;
}
ASSERT_EQ(it, iterable.end());
}
auto verify = [&](const std::optional<memgraph::utils::Bound<PropertyValue>> &from,
const std::optional<memgraph::utils::Bound<PropertyValue>> &to,
const std::vector<PropertyValue> &expected) {
auto acc = storage.Access();
auto iterable = acc.Vertices(label1, prop_val, from, to, View::OLD);
size_t i = 0;
for (auto it = iterable.begin(); it != iterable.end(); ++it, ++i) {
auto vertex = *it;
auto maybe_value = vertex.GetProperty(prop_val, View::OLD);
ASSERT_TRUE(maybe_value.HasValue());
ASSERT_EQ(*maybe_value, expected[i]);
}
ASSERT_EQ(i, expected.size());
};
// Range iteration with two specified bounds that have the same type should
// yield the naturally expected items.
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(false)),
memgraph::utils::MakeBoundExclusive(PropertyValue(true)), {});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(false)),
memgraph::utils::MakeBoundInclusive(PropertyValue(true)), {PropertyValue(true)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(false)),
memgraph::utils::MakeBoundExclusive(PropertyValue(true)), {PropertyValue(false)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(false)),
memgraph::utils::MakeBoundInclusive(PropertyValue(true)), {PropertyValue(false), PropertyValue(true)});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(0)), memgraph::utils::MakeBoundExclusive(PropertyValue(1.8)),
{PropertyValue(0.5), PropertyValue(1), PropertyValue(1.5)});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(0)), memgraph::utils::MakeBoundInclusive(PropertyValue(1.8)),
{PropertyValue(0.5), PropertyValue(1), PropertyValue(1.5)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(0)), memgraph::utils::MakeBoundExclusive(PropertyValue(1.8)),
{PropertyValue(0), PropertyValue(0.5), PropertyValue(1), PropertyValue(1.5)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(0)), memgraph::utils::MakeBoundInclusive(PropertyValue(1.8)),
{PropertyValue(0), PropertyValue(0.5), PropertyValue(1), PropertyValue(1.5)});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue("b")),
memgraph::utils::MakeBoundExclusive(PropertyValue("memgraph")), {PropertyValue("c")});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue("b")),
memgraph::utils::MakeBoundInclusive(PropertyValue("memgraph")), {PropertyValue("c")});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue("b")),
memgraph::utils::MakeBoundExclusive(PropertyValue("memgraph")), {PropertyValue("b"), PropertyValue("c")});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue("b")),
memgraph::utils::MakeBoundInclusive(PropertyValue("memgraph")), {PropertyValue("b"), PropertyValue("c")});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)})),
memgraph::utils::MakeBoundExclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue("b")})),
{PropertyValue(std::vector<PropertyValue>{PropertyValue(2)})});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)})),
memgraph::utils::MakeBoundInclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue("b")})),
{PropertyValue(std::vector<PropertyValue>{PropertyValue(2)})});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)})),
memgraph::utils::MakeBoundExclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue("b")})),
{PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(2)})});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)})),
memgraph::utils::MakeBoundInclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue("b")})),
{PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(2)})});
verify(memgraph::utils::MakeBoundExclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5.0)}})),
memgraph::utils::MakeBoundExclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue("b")}})),
{PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}})});
verify(memgraph::utils::MakeBoundExclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5.0)}})),
memgraph::utils::MakeBoundInclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue("b")}})),
{PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}})});
verify(memgraph::utils::MakeBoundInclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5.0)}})),
memgraph::utils::MakeBoundExclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue("b")}})),
{PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5)}}),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}})});
verify(memgraph::utils::MakeBoundInclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5.0)}})),
memgraph::utils::MakeBoundInclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue("b")}})),
{PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5)}}),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}})});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(temporals[0])),
memgraph::utils::MakeBoundInclusive(PropertyValue(TemporalData{TemporalType::Date, 200})),
// LocalDateTime has a "higher" type number so it is not part of the range
{PropertyValue(temporals[1])});
verify(memgraph::utils::MakeBoundExclusive(PropertyValue(temporals[0])),
memgraph::utils::MakeBoundInclusive(PropertyValue(temporals[2])),
{PropertyValue(temporals[1]), PropertyValue(temporals[2])});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(temporals[0])),
memgraph::utils::MakeBoundExclusive(PropertyValue(temporals[2])),
{PropertyValue(temporals[0]), PropertyValue(temporals[1])});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(temporals[0])),
memgraph::utils::MakeBoundInclusive(PropertyValue(temporals[2])),
{PropertyValue(temporals[0]), PropertyValue(temporals[1]), PropertyValue(temporals[2])});
// Range iteration with one unspecified bound should only yield items that
// have the same type as the specified bound.
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(false)), std::nullopt,
{PropertyValue(false), PropertyValue(true)});
verify(std::nullopt, memgraph::utils::MakeBoundExclusive(PropertyValue(true)), {PropertyValue(false)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(1)), std::nullopt,
{PropertyValue(1), PropertyValue(1.5), PropertyValue(2), PropertyValue(std::numeric_limits<int64_t>::max()),
PropertyValue(std::numeric_limits<double>::infinity())});
verify(std::nullopt, memgraph::utils::MakeBoundExclusive(PropertyValue(0)),
{PropertyValue(-std::numeric_limits<double>::infinity()), PropertyValue(std::numeric_limits<int64_t>::min()),
PropertyValue(-1), PropertyValue(-0.5)});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue("b")), std::nullopt,
{PropertyValue("b"), PropertyValue("c")});
verify(std::nullopt, memgraph::utils::MakeBoundExclusive(PropertyValue("b")),
{PropertyValue(""), PropertyValue("a")});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(false)})),
std::nullopt,
{PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(2)})});
verify(std::nullopt, memgraph::utils::MakeBoundExclusive(PropertyValue(std::vector<PropertyValue>{PropertyValue(1)})),
{PropertyValue(std::vector<PropertyValue>()), PropertyValue(std::vector<PropertyValue>{PropertyValue(0.8)})});
verify(memgraph::utils::MakeBoundInclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(false)}})),
std::nullopt,
{PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5)}}),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(10)}})});
verify(std::nullopt,
memgraph::utils::MakeBoundExclusive(
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(7.5)}})),
{PropertyValue(std::map<std::string, PropertyValue>()),
PropertyValue(std::map<std::string, PropertyValue>{{"id", PropertyValue(5)}})});
verify(memgraph::utils::MakeBoundInclusive(PropertyValue(TemporalData(TemporalType::Date, 10))), std::nullopt,
{PropertyValue(temporals[0]), PropertyValue(temporals[1]), PropertyValue(temporals[2])});
verify(std::nullopt, memgraph::utils::MakeBoundExclusive(PropertyValue(TemporalData(TemporalType::Duration, 0))),
{PropertyValue(temporals[0]), PropertyValue(temporals[1]), PropertyValue(temporals[2])});
// Range iteration with two specified bounds that don't have the same type
// should yield no items.
for (size_t i = 0; i < values.size(); ++i) {
for (size_t j = i; j < values.size(); ++j) {
if (PropertyValue::AreComparableTypes(values[i].type(), values[j].type())) {
verify(memgraph::utils::MakeBoundInclusive(values[i]), memgraph::utils::MakeBoundInclusive(values[j]),
{values.begin() + i, values.begin() + j + 1});
} else {
verify(memgraph::utils::MakeBoundInclusive(values[i]), memgraph::utils::MakeBoundInclusive(values[j]), {});
}
}
}
// Iteration without any bounds should return all items of the index.
verify(std::nullopt, std::nullopt, values);
}
TEST_F(IndexTest, LabelPropertyIndexCreateWithExistingPrimaryKey) {
// Create index on primary label and on primary key
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
EXPECT_FALSE(storage.CreateIndex(primary_label, primary_property));
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
// Create index on primary label and on secondary property
EXPECT_TRUE(storage.CreateIndex(primary_label, prop_id));
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 1);
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
{
auto acc = storage.Access();
EXPECT_TRUE(acc.LabelPropertyIndexExists(primary_label, prop_id));
}
EXPECT_THAT(storage.ListAllIndices().label_property, UnorderedElementsAre(Pair(primary_label, prop_id)));
// Create index on primary label
EXPECT_FALSE(storage.CreateIndex(primary_label));
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 1);
// Create index on secondary label
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_EQ(storage.ListAllIndices().label.size(), 1);
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 1);
}
TEST_F(IndexTest, LabelIndexCreateVertexAndValidate) {
{
auto acc = storage.Access();
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
}
{
auto acc = storage.Access();
// Create vertices with CreateVertexAndValidate
for (int i = 0; i < 5; ++i) {
auto vertex =
acc.CreateVertexAndValidate(primary_label, {label1}, {{primary_property, PropertyValue(primary_key_id++)}});
ASSERT_TRUE(vertex.HasValue());
}
ASSERT_NO_ERROR(acc.Commit());
}
{
EXPECT_TRUE(storage.CreateIndex(label1));
{
auto acc = storage.Access();
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
{
EXPECT_TRUE(storage.DropIndex(label1));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelIndexExists(label1));
}
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
}
{
auto acc = storage.Access();
EXPECT_TRUE(storage.CreateIndex(label1));
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
for (int i = 0; i < 5; ++i) {
auto vertex =
acc.CreateVertexAndValidate(primary_label, {label1}, {{primary_property, PropertyValue(primary_key_id++)}});
ASSERT_TRUE(vertex.HasValue());
}
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::NEW), View::NEW),
UnorderedElementsAre(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, View::OLD), View::OLD), UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
TEST_F(IndexTest, LabelPropertyIndexCreateVertexAndValidate) {
{
auto acc = storage.Access();
EXPECT_EQ(storage.ListAllIndices().label.size(), 0);
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
}
{
auto acc = storage.Access();
// Create vertices with CreateVertexAndValidate
for (int i = 0; i < 5; ++i) {
auto vertex = acc.CreateVertexAndValidate(
primary_label, {label1},
{{primary_property, PropertyValue(primary_key_id++)}, {prop_id, PropertyValue(vertex_id++)}});
ASSERT_TRUE(vertex.HasValue());
}
ASSERT_NO_ERROR(acc.Commit());
}
{
EXPECT_TRUE(storage.CreateIndex(label1, prop_id));
{
auto acc = storage.Access();
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::OLD), View::OLD),
UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
{
EXPECT_TRUE(storage.DropIndex(label1, prop_id));
{
auto acc = storage.Access();
EXPECT_FALSE(acc.LabelPropertyIndexExists(label1, prop_id));
}
EXPECT_EQ(storage.ListAllIndices().label_property.size(), 0);
}
{
auto acc = storage.Access();
EXPECT_TRUE(storage.CreateIndex(label1, prop_id));
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::OLD), View::OLD),
UnorderedElementsAre(0, 1, 2, 3, 4));
for (int i = 0; i < 5; ++i) {
auto vertex = acc.CreateVertexAndValidate(
primary_label, {label1},
{{primary_property, PropertyValue(primary_key_id++)}, {prop_id, PropertyValue(vertex_id++)}});
ASSERT_TRUE(vertex.HasValue());
}
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::NEW), View::NEW),
UnorderedElementsAre(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
EXPECT_THAT(GetPrimaryKeyIds(acc.Vertices(label1, prop_id, View::OLD), View::OLD),
UnorderedElementsAre(0, 1, 2, 3, 4));
}
}
} // namespace memgraph::storage::v3::tests

46
tests/unit/storage_v3_key_store.cpp Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <string>
#include <vector>
/**
* gtest/gtest.h must be included before rapidcheck/gtest.h!
*/
#include <gtest/gtest.h>
#include <rapidcheck.h>
#include <rapidcheck/gtest.h>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3::test {
RC_GTEST_PROP(KeyStore, KeyStore, (std::vector<std::string> values)) {
RC_PRE(!values.empty());
std::vector<PropertyValue> property_values;
property_values.reserve(values.size());
std::transform(values.begin(), values.end(), std::back_inserter(property_values),
[](std::string &value) { return PropertyValue{std::move(value)}; });
KeyStore key_store{property_values};
const auto keys = key_store.Keys();
RC_ASSERT(keys.size() == property_values.size());
for (int i = 0; i < keys.size(); ++i) {
RC_ASSERT(keys[i] == property_values[i]);
RC_ASSERT(key_store.GetKey(i) == property_values[i]);
}
}
} // namespace memgraph::storage::v3::test

622
tests/unit/storage_v3_property_store.cpp Normal file
View File

@ -0,0 +1,622 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <array>
#include <limits>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::storage::v3::tests {
class StorageV3PropertyStore : public ::testing::Test {
protected:
PropertyStore props;
const std::array<PropertyValue, 24> kSampleValues = {
PropertyValue(),
PropertyValue(false),
PropertyValue(true),
PropertyValue(0),
PropertyValue(33),
PropertyValue(-33),
PropertyValue(-3137),
PropertyValue(3137),
PropertyValue(310000007),
PropertyValue(-310000007),
PropertyValue(3100000000007L),
PropertyValue(-3100000000007L),
PropertyValue(0.0),
PropertyValue(33.33),
PropertyValue(-33.33),
PropertyValue(3137.3137),
PropertyValue(-3137.3137),
PropertyValue("sample"),
PropertyValue(std::string(404, 'n')),
PropertyValue(
std::vector<PropertyValue>{PropertyValue(33), PropertyValue(std::string("sample")), PropertyValue(-33.33)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(), PropertyValue(false)}),
PropertyValue(std::map<std::string, PropertyValue>{{"sample", PropertyValue()}, {"key", PropertyValue(false)}}),
PropertyValue(std::map<std::string, PropertyValue>{
{"test", PropertyValue(33)}, {"map", PropertyValue(std::string("sample"))}, {"item", PropertyValue(-33.33)}}),
PropertyValue(TemporalData(TemporalType::Date, 23)),
};
void AssertPropertyIsEqual(const PropertyStore &store, PropertyId property, const PropertyValue &value) {
ASSERT_TRUE(store.IsPropertyEqual(property, value));
for (const auto &sample : kSampleValues) {
if (sample == value) {
ASSERT_TRUE(store.IsPropertyEqual(property, sample));
} else {
ASSERT_FALSE(store.IsPropertyEqual(property, sample));
}
}
}
};
using testing::UnorderedElementsAre;
TEST_F(StorageV3PropertyStore, StoreTwoProperties) {
const auto make_prop = [](int64_t prop_id_and_value) {
auto prop = PropertyId::FromInt(prop_id_and_value);
auto value = PropertyValue(prop_id_and_value);
return std::make_pair(prop, value);
};
const auto first_prop_and_value = make_prop(42);
const auto second_prop_and_value = make_prop(43);
ASSERT_TRUE(props.SetProperty(first_prop_and_value.first, first_prop_and_value.second));
ASSERT_TRUE(props.SetProperty(second_prop_and_value.first, second_prop_and_value.second));
ASSERT_THAT(props.Properties(), UnorderedElementsAre(first_prop_and_value, second_prop_and_value));
}
TEST_F(StorageV3PropertyStore, Simple) {
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, SimpleLarge) {
auto prop = PropertyId::FromInt(42);
{
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
{
auto value = PropertyValue(TemporalData(TemporalType::Date, 23));
ASSERT_FALSE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
ASSERT_FALSE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptySetToNull) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, Clear) {
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_TRUE(props.ClearProperties());
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptyClear) {
ASSERT_FALSE(props.ClearProperties());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveConstruct) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
PropertyStore props2(std::move(props1));
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveConstructLarge) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
PropertyStore props2(std::move(props1));
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveAssign) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
auto value2 = PropertyValue(68);
PropertyStore props2;
ASSERT_TRUE(props2.SetProperty(prop, value2));
ASSERT_EQ(props2.GetProperty(prop), value2);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value2);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value2)));
props2 = std::move(props1);
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveAssignLarge) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
auto value2 = PropertyValue(std::string(10000, 'b'));
PropertyStore props2;
ASSERT_TRUE(props2.SetProperty(prop, value2));
ASSERT_EQ(props2.GetProperty(prop), value2);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value2);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value2)));
props2 = std::move(props1);
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptySet) {
std::vector<PropertyValue> vec{PropertyValue(true), PropertyValue(123), PropertyValue()};
std::map<std::string, PropertyValue> map{{"nandare", PropertyValue(false)}};
const TemporalData temporal{TemporalType::LocalDateTime, 23};
std::vector<PropertyValue> data{PropertyValue(true), PropertyValue(123), PropertyValue(123.5),
PropertyValue("nandare"), PropertyValue(vec), PropertyValue(map),
PropertyValue(temporal)};
auto prop = PropertyId::FromInt(42);
for (const auto &value : data) {
PropertyStore local_props;
ASSERT_TRUE(local_props.SetProperty(prop, value));
ASSERT_EQ(local_props.GetProperty(prop), value);
ASSERT_TRUE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, value);
ASSERT_THAT(local_props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(local_props.SetProperty(prop, value));
ASSERT_EQ(local_props.GetProperty(prop), value);
ASSERT_TRUE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, value);
ASSERT_THAT(local_props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(local_props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(local_props.GetProperty(prop).IsNull());
ASSERT_FALSE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, PropertyValue());
ASSERT_EQ(local_props.Properties().size(), 0);
ASSERT_TRUE(local_props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(local_props.GetProperty(prop).IsNull());
ASSERT_FALSE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, PropertyValue());
ASSERT_EQ(local_props.Properties().size(), 0);
}
}
TEST_F(StorageV3PropertyStore, FullSet) {
std::vector<PropertyValue> vec{PropertyValue(true), PropertyValue(123), PropertyValue()};
std::map<std::string, PropertyValue> map{{"nandare", PropertyValue(false)}};
const TemporalData temporal{TemporalType::LocalDateTime, 23};
std::map<PropertyId, PropertyValue> data{
{PropertyId::FromInt(1), PropertyValue(true)}, {PropertyId::FromInt(2), PropertyValue(123)},
{PropertyId::FromInt(3), PropertyValue(123.5)}, {PropertyId::FromInt(4), PropertyValue("nandare")},
{PropertyId::FromInt(5), PropertyValue(vec)}, {PropertyId::FromInt(6), PropertyValue(map)},
{PropertyId::FromInt(7), PropertyValue(temporal)}};
std::vector<PropertyValue> alt{PropertyValue(),
PropertyValue(std::string()),
PropertyValue(std::string(10, 'a')),
PropertyValue(std::string(100, 'a')),
PropertyValue(std::string(1000, 'a')),
PropertyValue(std::string(10000, 'a')),
PropertyValue(std::string(100000, 'a'))};
for (const auto &target : data) {
for (const auto &item : data) {
ASSERT_TRUE(props.SetProperty(item.first, item.second));
}
for (size_t i = 0; i < alt.size(); ++i) {
if (i == 1) {
ASSERT_TRUE(props.SetProperty(target.first, alt[i]));
} else {
ASSERT_FALSE(props.SetProperty(target.first, alt[i]));
}
for (const auto &item : data) {
if (item.first == target.first) {
ASSERT_EQ(props.GetProperty(item.first), alt[i]);
if (alt[i].IsNull()) {
ASSERT_FALSE(props.HasProperty(item.first));
} else {
ASSERT_TRUE(props.HasProperty(item.first));
}
AssertPropertyIsEqual(props, item.first, alt[i]);
} else {
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
}
auto current = data;
if (alt[i].IsNull()) {
current.erase(target.first);
} else {
current[target.first] = alt[i];
}
ASSERT_EQ(props.Properties(), current);
}
for (ssize_t i = alt.size() - 1; i >= 0; --i) {
ASSERT_FALSE(props.SetProperty(target.first, alt[i]));
for (const auto &item : data) {
if (item.first == target.first) {
ASSERT_EQ(props.GetProperty(item.first), alt[i]);
if (alt[i].IsNull()) {
ASSERT_FALSE(props.HasProperty(item.first));
} else {
ASSERT_TRUE(props.HasProperty(item.first));
}
AssertPropertyIsEqual(props, item.first, alt[i]);
} else {
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
}
auto current = data;
if (alt[i].IsNull()) {
current.erase(target.first);
} else {
current[target.first] = alt[i];
}
ASSERT_EQ(props.Properties(), current);
}
ASSERT_TRUE(props.SetProperty(target.first, target.second));
ASSERT_EQ(props.GetProperty(target.first), target.second);
ASSERT_TRUE(props.HasProperty(target.first));
AssertPropertyIsEqual(props, target.first, target.second);
props.ClearProperties();
ASSERT_EQ(props.Properties().size(), 0);
for (const auto &item : data) {
ASSERT_TRUE(props.GetProperty(item.first).IsNull());
ASSERT_FALSE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, PropertyValue());
}
}
}
TEST_F(StorageV3PropertyStore, IntEncoding) {
std::map<PropertyId, PropertyValue> data{
// {PropertyId::FromUint(0UL),
// PropertyValue(std::numeric_limits<int64_t>::min())},
// {PropertyId::FromUint(10UL), PropertyValue(-137438953472L)},
// {PropertyId::FromUint(std::numeric_limits<uint8_t>::max()),
// PropertyValue(-4294967297L)},
// {PropertyId::FromUint(256UL),
// PropertyValue(std::numeric_limits<int32_t>::min())},
// {PropertyId::FromUint(1024UL), PropertyValue(-1048576L)},
// {PropertyId::FromUint(1025UL), PropertyValue(-65537L)},
// {PropertyId::FromUint(1026UL),
// PropertyValue(std::numeric_limits<int16_t>::min())},
// {PropertyId::FromUint(1027UL), PropertyValue(-1024L)},
// {PropertyId::FromUint(2000UL), PropertyValue(-257L)},
// {PropertyId::FromUint(3000UL),
// PropertyValue(std::numeric_limits<int8_t>::min())},
// {PropertyId::FromUint(4000UL), PropertyValue(-1L)},
// {PropertyId::FromUint(10000UL), PropertyValue(0L)},
// {PropertyId::FromUint(20000UL), PropertyValue(1L)},
// {PropertyId::FromUint(30000UL),
// PropertyValue(std::numeric_limits<int8_t>::max())},
// {PropertyId::FromUint(40000UL), PropertyValue(256L)},
// {PropertyId::FromUint(50000UL), PropertyValue(1024L)},
// {PropertyId::FromUint(std::numeric_limits<uint16_t>::max()),
// PropertyValue(std::numeric_limits<int16_t>::max())},
// {PropertyId::FromUint(65536UL), PropertyValue(65536L)},
// {PropertyId::FromUint(1048576UL), PropertyValue(1048576L)},
// {PropertyId::FromUint(std::numeric_limits<uint32_t>::max()),
// PropertyValue(std::numeric_limits<int32_t>::max())},
{PropertyId::FromUint(4294967296UL), PropertyValue(4294967296L)},
{PropertyId::FromUint(137438953472UL), PropertyValue(137438953472L)},
{PropertyId::FromUint(std::numeric_limits<uint64_t>::max()), PropertyValue(std::numeric_limits<int64_t>::max())}};
for (const auto &item : data) {
ASSERT_TRUE(props.SetProperty(item.first, item.second));
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
for (auto it = data.rbegin(); it != data.rend(); ++it) {
const auto &item = *it;
ASSERT_FALSE(props.SetProperty(item.first, item.second)) << item.first.AsInt();
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
ASSERT_EQ(props.Properties(), data);
props.ClearProperties();
ASSERT_EQ(props.Properties().size(), 0);
for (const auto &item : data) {
ASSERT_TRUE(props.GetProperty(item.first).IsNull());
ASSERT_FALSE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, PropertyValue());
}
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualIntAndDouble) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(42)));
std::vector<std::pair<PropertyValue, PropertyValue>> tests{
{PropertyValue(0), PropertyValue(0.0)},
{PropertyValue(123), PropertyValue(123.0)},
{PropertyValue(12345), PropertyValue(12345.0)},
{PropertyValue(12345678), PropertyValue(12345678.0)},
{PropertyValue(1234567890123L), PropertyValue(1234567890123.0)},
};
// Test equality with raw values.
for (auto test : tests) {
ASSERT_EQ(test.first, test.second);
// Test first, second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test second, first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
// Make both negative
test.first = PropertyValue(test.first.ValueInt() * -1);
test.second = PropertyValue(test.second.ValueDouble() * -1.0);
ASSERT_EQ(test.first, test.second);
// Test -first, -second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test -second, -first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
}
// Test equality with values wrapped in lists.
for (auto test : tests) {
test.first = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.first.ValueInt())});
test.second = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.second.ValueDouble())});
ASSERT_EQ(test.first, test.second);
// Test first, second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test second, first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
// Make both negative
test.first = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.first.ValueList()[0].ValueInt() * -1)});
test.second =
PropertyValue(std::vector<PropertyValue>{PropertyValue(test.second.ValueList()[0].ValueDouble() * -1.0)});
ASSERT_EQ(test.first, test.second);
// Test -first, -second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test -second, -first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
}
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualString) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue("test")));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue("test")));
// Different length.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("helloworld")));
// Same length, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("asdf")));
// Shortened and extended.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("tes")));
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("testt")));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualList) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(
props.SetProperty(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test")})));
ASSERT_TRUE(
props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test")})));
// Different length.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(24)})));
// Same length, different value.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("asdf")})));
// Shortened and extended.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42)})));
ASSERT_FALSE(props.IsPropertyEqual(
prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test"), PropertyValue(true)})));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualMap) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("test")}})));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("test")}})));
// Different length.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{{"fgh", PropertyValue(24)}})));
// Same length, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("testt")}})));
// Same length, different key (different length).
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyxw", PropertyValue("test")}})));
// Same length, different key (same length).
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyw", PropertyValue("test")}})));
// Shortened and extended.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{{"abc", PropertyValue(42)}})));
ASSERT_FALSE(props.IsPropertyEqual(
prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"sdf", PropertyValue(true)}, {"zyx", PropertyValue("test")}})));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualTemporalData) {
auto prop = PropertyId::FromInt(42);
const TemporalData temporal{TemporalType::Date, 23};
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(temporal)));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue(temporal)));
// Different type.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(TemporalData{TemporalType::Duration, 23})));
// Same type, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(TemporalData{TemporalType::Date, 30})));
}
} // namespace memgraph::storage::v3::tests

4
tests/unit/storage_v3_schema.cpp
View File

@ -281,13 +281,13 @@ TEST_F(SchemaValidatorTest, TestSchemaValidatePropertyUpdateLabel) {
const auto schema_violation = schema_validator.ValidateLabelUpdate(label1);
ASSERT_NE(schema_violation, std::nullopt);
EXPECT_EQ(*schema_violation,
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_MODIFY_PRIMARY_LABEL, label1));
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label1));
}
{
const auto schema_violation = schema_validator.ValidateLabelUpdate(label2);
ASSERT_NE(schema_violation, std::nullopt);
EXPECT_EQ(*schema_violation,
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_MODIFY_PRIMARY_LABEL, label2));
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label2));
}
EXPECT_EQ(schema_validator.ValidateLabelUpdate(NameToLabel("test")), std::nullopt);
}

23
tests/unit/storage_v3_test_utils.cpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage_v3_test_utils.hpp"
namespace memgraph::storage::v3::tests {
size_t CountVertices(Shard::Accessor &storage_accessor, View view) {
auto vertices = storage_accessor.Vertices(view);
size_t count = 0U;
for (auto it = vertices.begin(); it != vertices.end(); ++it, ++count)
;
return count;
}
} // namespace memgraph::storage::v3::tests

21
tests/unit/storage_v3_test_utils.hpp Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/storage.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3::tests {
size_t CountVertices(Shard::Accessor &storage_accessor, View view);
} // namespace memgraph::storage::v3::tests

217
tests/unit/storage_v3_vertex_accessors.cpp Normal file
View File

@ -0,0 +1,217 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <limits>
#include <variant>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "common/types.hpp"
#include "storage/v3/delta.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage_v3_test_utils.hpp"
using testing::UnorderedElementsAre;
namespace memgraph::storage::v3::tests {
class StorageV3Accessor : public ::testing::Test {
protected:
void SetUp() override {
ASSERT_TRUE(storage.CreateSchema(primary_label, {SchemaProperty{primary_property, common::SchemaType::INT}}));
}
VertexAccessor CreateVertexAndValidate(Shard::Accessor &acc, LabelId primary_label,
const std::vector<LabelId> &labels,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties) {
auto vtx = acc.CreateVertexAndValidate(primary_label, labels, properties);
EXPECT_TRUE(vtx.HasValue());
return *vtx;
}
LabelId NameToLabelId(std::string_view label_name) { return LabelId::FromUint(id_mapper.NameToId(label_name)); }
PropertyId NameToPropertyId(std::string_view property_name) {
return PropertyId::FromUint(id_mapper.NameToId(property_name));
}
const std::vector<PropertyValue> pk{PropertyValue{0}};
NameIdMapper id_mapper;
Shard storage{NameToLabelId("label"), pk, std::nullopt};
const LabelId primary_label{NameToLabelId("label")};
const PropertyId primary_property{NameToPropertyId("property")};
};
TEST_F(StorageV3Accessor, TestPrimaryLabel) {
{
auto acc = storage.Access();
const auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(0)}});
ASSERT_TRUE(vertex.PrimaryLabel(View::NEW).HasValue());
const auto vertex_primary_label = vertex.PrimaryLabel(View::NEW).GetValue();
ASSERT_FALSE(vertex.PrimaryLabel(View::OLD).HasValue());
EXPECT_EQ(vertex_primary_label, primary_label);
}
{
auto acc = storage.Access();
const auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}});
ASSERT_TRUE(vertex.PrimaryLabel(View::OLD).HasError());
const auto error_primary_label = vertex.PrimaryLabel(View::OLD).GetError();
ASSERT_FALSE(vertex.PrimaryLabel(View::NEW).HasError());
EXPECT_EQ(error_primary_label, Error::NONEXISTENT_OBJECT);
}
{
auto acc = storage.Access();
CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}});
ASSERT_FALSE(acc.Commit().HasError());
}
{
auto acc = storage.Access();
const auto vertex = acc.FindVertex({PropertyValue{2}}, View::OLD);
ASSERT_TRUE(vertex.has_value());
ASSERT_TRUE(acc.FindVertex({PropertyValue{2}}, View::NEW).has_value());
ASSERT_TRUE(vertex->PrimaryLabel(View::NEW).HasValue());
ASSERT_TRUE(vertex->PrimaryLabel(View::OLD).HasValue());
const auto vertex_primary_label = vertex->PrimaryLabel(View::NEW).GetValue();
EXPECT_EQ(vertex_primary_label, primary_label);
}
}
TEST_F(StorageV3Accessor, TestAddLabels) {
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label1");
const auto label2 = NameToLabelId("label2");
const auto label3 = NameToLabelId("label3");
const auto vertex =
CreateVertexAndValidate(acc, primary_label, {label1, label2, label3}, {{primary_property, PropertyValue(0)}});
ASSERT_TRUE(vertex.Labels(View::NEW).HasValue());
ASSERT_FALSE(vertex.Labels(View::OLD).HasValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label2, label3));
}
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label1");
const auto label2 = NameToLabelId("label2");
const auto label3 = NameToLabelId("label3");
auto vertex = CreateVertexAndValidate(acc, primary_label, {label1}, {{primary_property, PropertyValue(1)}});
ASSERT_TRUE(vertex.Labels(View::NEW).HasValue());
ASSERT_FALSE(vertex.Labels(View::OLD).HasValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1));
EXPECT_TRUE(vertex.AddLabelAndValidate(label2).HasValue());
EXPECT_TRUE(vertex.AddLabelAndValidate(label3).HasValue());
ASSERT_TRUE(vertex.Labels(View::NEW).HasValue());
ASSERT_FALSE(vertex.Labels(View::OLD).HasValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label2, label3));
}
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label");
auto vertex = acc.CreateVertexAndValidate(primary_label, {label1}, {{primary_property, PropertyValue(2)}});
ASSERT_TRUE(vertex.HasError());
ASSERT_TRUE(std::holds_alternative<SchemaViolation>(vertex.GetError()));
EXPECT_EQ(std::get<SchemaViolation>(vertex.GetError()),
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, label1));
}
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label");
auto vertex = acc.CreateVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(3)}});
ASSERT_TRUE(vertex.HasValue());
const auto schema_violation = vertex->AddLabelAndValidate(label1);
ASSERT_TRUE(schema_violation.HasError());
ASSERT_TRUE(std::holds_alternative<SchemaViolation>(schema_violation.GetError()));
EXPECT_EQ(std::get<SchemaViolation>(schema_violation.GetError()),
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label1));
}
}
TEST_F(StorageV3Accessor, TestRemoveLabels) {
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label1");
const auto label2 = NameToLabelId("label2");
const auto label3 = NameToLabelId("label3");
auto vertex =
CreateVertexAndValidate(acc, primary_label, {label1, label2, label3}, {{primary_property, PropertyValue(0)}});
ASSERT_TRUE(vertex.Labels(View::NEW).HasValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label2, label3));
const auto res1 = vertex.RemoveLabelAndValidate(label2);
ASSERT_TRUE(res1.HasValue());
EXPECT_TRUE(res1.GetValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label1, label3));
const auto res2 = vertex.RemoveLabelAndValidate(label1);
ASSERT_TRUE(res2.HasValue());
EXPECT_TRUE(res2.GetValue());
EXPECT_THAT(vertex.Labels(View::NEW).GetValue(), UnorderedElementsAre(label3));
const auto res3 = vertex.RemoveLabelAndValidate(label3);
ASSERT_TRUE(res3.HasValue());
ASSERT_TRUE(res3.HasValue());
EXPECT_TRUE(res3.GetValue());
EXPECT_TRUE(vertex.Labels(View::NEW).GetValue().empty());
}
{
auto acc = storage.Access();
const auto label1 = NameToLabelId("label1");
auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}});
ASSERT_TRUE(vertex.Labels(View::NEW).HasValue());
EXPECT_TRUE(vertex.Labels(View::NEW).GetValue().empty());
const auto res1 = vertex.RemoveLabelAndValidate(label1);
ASSERT_TRUE(res1.HasValue());
EXPECT_FALSE(res1.GetValue());
}
{
auto acc = storage.Access();
auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}});
const auto res1 = vertex.RemoveLabelAndValidate(primary_label);
ASSERT_TRUE(res1.HasError());
ASSERT_TRUE(std::holds_alternative<SchemaViolation>(res1.GetError()));
EXPECT_EQ(std::get<SchemaViolation>(res1.GetError()),
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, primary_label));
}
}
TEST_F(StorageV3Accessor, TestSetKeysAndProperties) {
{
auto acc = storage.Access();
const PropertyId prop1{NameToPropertyId("prop1")};
auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(0)}});
const auto res = vertex.SetPropertyAndValidate(prop1, PropertyValue(1));
ASSERT_TRUE(res.HasValue());
}
{
auto acc = storage.Access();
auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(1)}});
const auto res = vertex.SetPropertyAndValidate(primary_property, PropertyValue(1));
ASSERT_TRUE(res.HasError());
ASSERT_TRUE(std::holds_alternative<SchemaViolation>(res.GetError()));
EXPECT_EQ(std::get<SchemaViolation>(res.GetError()),
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY, primary_label,
SchemaProperty{primary_property, common::SchemaType::INT}));
}
{
auto acc = storage.Access();
auto vertex = CreateVertexAndValidate(acc, primary_label, {}, {{primary_property, PropertyValue(2)}});
const auto res = vertex.SetPropertyAndValidate(primary_property, PropertyValue());
ASSERT_TRUE(res.HasError());
ASSERT_TRUE(std::holds_alternative<SchemaViolation>(res.GetError()));
EXPECT_EQ(std::get<SchemaViolation>(res.GetError()),
SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY, primary_label,
SchemaProperty{primary_property, common::SchemaType::INT}));
}
}
} // namespace memgraph::storage::v3::tests