tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make storage use KeyStore (#455)

Browse Source
This commit is contained in:
János Benjamin Antal 2022-08-03 18:10:58 +02:00 committed by GitHub
parent cc5ee6a496
commit 2891041468
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 3629 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/storage/v3/CMakeLists.txt
View File

@ -8,6 +8,7 @@ set(storage_v3_src_files
durability/wal.cpp
edge_accessor.cpp
indices.cpp
key_store.cpp
property_store.cpp
vertex_accessor.cpp
storage.cpp)

5
src/storage/v3/constraints.cpp
View File

@ -279,7 +279,7 @@ void UniqueConstraints::UpdateBeforeCommit(const Vertex *vertex, const Transacti
}
utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> UniqueConstraints::CreateConstraint(
LabelId label, const std::set<PropertyId> &properties, utils::SkipList<Vertex>::Accessor vertices) {
LabelId label, const std::set<PropertyId> &properties, VerticesSkipList::Accessor vertices) {
if (properties.empty()) {
return CreationStatus::EMPTY_PROPERTIES;
}
@ -300,7 +300,8 @@ utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus> Uniqu
{
auto acc = constraint->second.access();
for (const Vertex &vertex : vertices) {
for (const auto &lo_vertex : vertices) {
const auto &vertex = lo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}

12
src/storage/v3/constraints.hpp
View File

@ -18,6 +18,7 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
#include "utils/skip_list.hpp"
@ -108,7 +109,7 @@ class UniqueConstraints {
/// @throw std::bad_alloc
utils::BasicResult<ConstraintViolation, CreationStatus> CreateConstraint(LabelId label,
const std::set<PropertyId> &properties,
utils::SkipList<Vertex>::Accessor vertices);
VerticesSkipList::Accessor vertices);
/// Deletes the specified constraint. Returns `DeletionStatus::NOT_FOUND` if
/// there is not such constraint in the storage,
@ -152,12 +153,15 @@ struct Constraints {
///
/// @throw std::bad_alloc
/// @throw std::length_error
inline utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(
Constraints *constraints, LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices) {
inline utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(Constraints *constraints, LabelId label,
PropertyId property,
VerticesSkipList::Accessor vertices) {
if (utils::Contains(constraints->existence_constraints, std::make_pair(label, property))) {
return false;
}
for (const auto &vertex : vertices) {
for (const auto &lgo_vertex : vertices) {
const auto &vertex = lgo_vertex.vertex;
if (!vertex.deleted && utils::Contains(vertex.labels, label) && !vertex.properties.HasProperty(property)) {
return ConstraintViolation{ConstraintViolation::Type::EXISTENCE, label, std::set<PropertyId>{property}};
}

15
src/storage/v3/durability/durability.cpp
View File

@ -113,7 +113,7 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
// to ensure that the indices and constraints are consistent at the end of the
// recovery process.
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
Constraints *constraints, utils::SkipList<Vertex> *vertices) {
Constraints *constraints, VerticesSkipList *vertices) {
spdlog::info("Recreating indices from metadata.");
// Recover label indices.
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
@ -157,14 +157,11 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
spdlog::info("Constraints are recreated from metadata.");
}
std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
uint64_t *wal_seq_num) {
std::optional<RecoveryInfo> RecoverData(
const std::filesystem::path &snapshot_directory, const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id, std::deque<std::pair<std::string, uint64_t>> *epoch_history, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper, Indices *indices,
Constraints *constraints, Config::Items items, uint64_t *wal_seq_num) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
wal_directory);

15
src/storage/v3/durability/durability.hpp
View File

@ -97,18 +97,15 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
// recovery process.
/// @throw RecoveryFailure
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
Constraints *constraints, utils::SkipList<Vertex> *vertices);
Constraints *constraints, VerticesSkipList *vertices);
/// Recovers data either from a snapshot and/or WAL files.
/// @throw RecoveryFailure
/// @throw std::bad_alloc
std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items,
uint64_t *wal_seq_num);
std::optional<RecoveryInfo> RecoverData(
const std::filesystem::path &snapshot_directory, const std::filesystem::path &wal_directory, std::string *uuid,
std::string *epoch_id, std::deque<std::pair<std::string, uint64_t>> *epoch_history, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper, Indices *indices,
Constraints *constraints, Config::Items items, uint64_t *wal_seq_num);
} // namespace memgraph::storage::v3::durability

40
src/storage/v3/durability/snapshot.cpp
View File

@ -20,6 +20,7 @@
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
@ -157,7 +158,7 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
return info;
}
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items) {
@ -305,7 +306,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
last_vertex_gid = *gid;
spdlog::debug("Recovering vertex {}.", *gid);
auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
auto [it, inserted] = vertex_acc.insert({Vertex{Gid::FromUint(*gid), nullptr}});
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
// Recover labels.
@ -313,7 +314,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
{
auto labels_size = snapshot.ReadUint();
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
auto &labels = it->labels;
auto &labels = it->vertex.labels;
labels.reserve(*labels_size);
for (uint64_t j = 0; j < *labels_size; ++j) {
auto label = snapshot.ReadUint();
@ -329,7 +330,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
{
auto props_size = snapshot.ReadUint();
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
auto &props = it->properties;
auto &props = it->vertex.properties;
for (uint64_t j = 0; j < *props_size; ++j) {
auto key = snapshot.ReadUint();
if (!key) throw RecoveryFailure("Invalid snapshot data!");
@ -372,17 +373,18 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover vertices (in/out edges).
spdlog::info("Recovering connectivity.");
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
for (auto &vertex : vertex_acc) {
for (auto &lgo_vertex : vertex_acc) {
auto &vertex = lgo_vertex.vertex;
{
auto marker = snapshot.ReadMarker();
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
}
spdlog::trace("Recovering connectivity for vertex {}.", vertex.gid.AsUint());
spdlog::trace("Recovering connectivity for vertex {}.", vertex.Gid().AsUint());
// Check vertex.
auto gid = snapshot.ReadUint();
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
if (gid != vertex.gid.AsUint()) throw RecoveryFailure("Invalid snapshot data!");
if (gid != vertex.Gid().AsUint()) throw RecoveryFailure("Invalid snapshot data!");
// Skip labels.
{
@ -408,7 +410,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover in edges.
{
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.Gid().AsUint());
auto in_size = snapshot.ReadUint();
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.in_edges.reserve(*in_size);
@ -422,7 +424,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
auto edge_type = snapshot.ReadUint();
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
auto from_vertex = vertex_acc.find(Gid::FromUint(*from_gid));
auto from_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*from_gid).AsInt()}});
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid from vertex!");
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
@ -437,14 +439,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
}
SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->gid.AsUint());
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->vertex.Gid().AsUint());
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &from_vertex->vertex, edge_ref);
}
}
// Recover out edges.
{
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.Gid().AsUint());
auto out_size = snapshot.ReadUint();
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.out_edges.reserve(*out_size);
@ -458,7 +460,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
auto edge_type = snapshot.ReadUint();
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
auto to_vertex = vertex_acc.find(Gid::FromUint(*to_gid));
auto to_vertex = vertex_acc.find(std::vector{PropertyValue{Gid::FromUint(*to_gid).AsInt()}});
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid to vertex!");
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
@ -473,8 +475,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
}
SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->gid.AsUint());
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->vertex.Gid().AsUint());
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &to_vertex->vertex, edge_ref);
}
// Increment edge count. We only increment the count here because the
// information is duplicated in in_edges.
@ -627,7 +629,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items, const std::string &uuid,
const std::string_view epoch_id, const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
utils::FileRetainer *file_retainer) {
@ -740,9 +742,9 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
{
offset_vertices = snapshot.GetPosition();
auto acc = vertices->access();
for (auto &vertex : acc) {
for (auto &lgo_vertex : acc) {
// The visibility check is implemented for vertices so we use it here.
auto va = VertexAccessor::Create(&vertex, transaction, indices, constraints, items, View::OLD);
auto va = VertexAccessor::Create(&lgo_vertex.vertex, transaction, indices, constraints, items, View::OLD);
if (!va) continue;
// Get vertex data.
@ -760,7 +762,7 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
// Store the vertex.
{
snapshot.WriteMarker(Marker::SECTION_VERTEX);
snapshot.WriteUint(vertex.gid.AsUint());
snapshot.WriteUint(lgo_vertex.vertex.Gid().AsUint());
const auto &labels = maybe_labels.GetValue();
snapshot.WriteUint(labels.size());
for (const auto &item : labels) {

4
src/storage/v3/durability/snapshot.hpp
View File

@ -59,7 +59,7 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
/// Function used to load the snapshot data into the storage.
/// @throw RecoveryFailure
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges,
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items);
@ -67,7 +67,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
/// Function used to create a snapshot using the given transaction.
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
VerticesSkipList *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
Indices *indices, Constraints *constraints, Config::Items items, const std::string &uuid,
std::string_view epoch_id, const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
utils::FileRetainer *file_retainer);

119
src/storage/v3/durability/wal.cpp
View File

@ -17,6 +17,7 @@
#include "storage/v3/durability/version.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
@ -492,12 +493,12 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Ite
case Delta::Action::DELETE_OBJECT:
case Delta::Action::RECREATE_OBJECT: {
encoder->WriteMarker(VertexActionToMarker(delta.action));
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteUint(vertex.Gid().AsUint());
break;
}
case Delta::Action::SET_PROPERTY: {
encoder->WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY);
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteUint(vertex.Gid().AsUint());
encoder->WriteString(name_id_mapper->IdToName(delta.property.key.AsUint()));
// The property value is the value that is currently stored in the
// vertex.
@ -510,7 +511,7 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Ite
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL: {
encoder->WriteMarker(VertexActionToMarker(delta.action));
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteUint(vertex.Gid().AsUint());
encoder->WriteString(name_id_mapper->IdToName(delta.label.AsUint()));
break;
}
@ -523,8 +524,8 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Ite
encoder->WriteUint(delta.vertex_edge.edge.gid.AsUint());
}
encoder->WriteString(name_id_mapper->IdToName(delta.vertex_edge.edge_type.AsUint()));
encoder->WriteUint(vertex.gid.AsUint());
encoder->WriteUint(delta.vertex_edge.vertex->gid.AsUint());
encoder->WriteUint(vertex.Gid().AsUint());
encoder->WriteUint(delta.vertex_edge.vertex->Gid().AsUint());
break;
}
case Delta::Action::ADD_IN_EDGE:
@ -617,7 +618,7 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Storage
}
RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints,
const std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
const std::optional<uint64_t> last_loaded_timestamp, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
Config::Items items) {
spdlog::info("Trying to load WAL file {}.", path);
@ -653,7 +654,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
auto delta = ReadWalDeltaData(&wal);
switch (delta.type) {
case WalDeltaData::Type::VERTEX_CREATE: {
auto [vertex, inserted] = vertex_acc.insert(Vertex{delta.vertex_create_delete.gid, nullptr});
auto [vertex, inserted] = vertex_acc.insert({Vertex{delta.vertex_create_delete.gid, nullptr}});
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
ret.next_vertex_id = std::max(ret.next_vertex_id, delta.vertex_create_delete.gid.AsUint() + 1);
@ -661,51 +662,66 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
break;
}
case WalDeltaData::Type::VERTEX_DELETE: {
auto vertex = vertex_acc.find(delta.vertex_create_delete.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
if (!vertex->in_edges.empty() || !vertex->out_edges.empty())
auto lgo_vertex_it = vertex_acc.find(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}});
if (lgo_vertex_it == vertex_acc.end()) {
throw RecoveryFailure("The vertex doesn't exist!");
}
auto &vertex = lgo_vertex_it->vertex;
if (!vertex.in_edges.empty() || !vertex.out_edges.empty())
throw RecoveryFailure("The vertex can't be deleted because it still has edges!");
if (!vertex_acc.remove(delta.vertex_create_delete.gid))
if (!vertex_acc.remove(std::vector{PropertyValue{delta.vertex_create_delete.gid.AsInt()}}))
throw RecoveryFailure("The vertex must be removed here!");
break;
}
case WalDeltaData::Type::VERTEX_ADD_LABEL:
case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
auto vertex = vertex_acc.find(delta.vertex_add_remove_label.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
auto lgo_vertex_it = vertex_acc.find(std::vector{PropertyValue{delta.vertex_add_remove_label.gid.AsInt()}});
if (lgo_vertex_it == vertex_acc.end()) {
throw RecoveryFailure("The vertex doesn't exist!");
}
auto &vertex = lgo_vertex_it->vertex;
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.vertex_add_remove_label.label));
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), label_id);
auto it = std::find(vertex.labels.begin(), vertex.labels.end(), label_id);
if (delta.type == WalDeltaData::Type::VERTEX_ADD_LABEL) {
if (it != vertex->labels.end()) throw RecoveryFailure("The vertex already has the label!");
vertex->labels.push_back(label_id);
if (it != vertex.labels.end()) throw RecoveryFailure("The vertex already has the label!");
vertex.labels.push_back(label_id);
} else {
if (it == vertex->labels.end()) throw RecoveryFailure("The vertex doesn't have the label!");
std::swap(*it, vertex->labels.back());
vertex->labels.pop_back();
if (it == vertex.labels.end()) throw RecoveryFailure("The vertex doesn't have the label!");
std::swap(*it, vertex.labels.back());
vertex.labels.pop_back();
}
break;
}
case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
auto vertex = vertex_acc.find(delta.vertex_edge_set_property.gid);
if (vertex == vertex_acc.end()) throw RecoveryFailure("The vertex doesn't exist!");
auto lgo_vertex_it = vertex_acc.find(std::vector{PropertyValue{delta.vertex_edge_set_property.gid.AsInt()}});
if (lgo_vertex_it == vertex_acc.end()) {
throw RecoveryFailure("The vertex doesn't exist!");
}
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.vertex_edge_set_property.property));
auto &property_value = delta.vertex_edge_set_property.value;
vertex->properties.SetProperty(property_id, property_value);
lgo_vertex_it->vertex.properties.SetProperty(property_id, property_value);
break;
}
case WalDeltaData::Type::EDGE_CREATE: {
auto from_vertex = vertex_acc.find(delta.edge_create_delete.from_vertex);
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("The from vertex doesn't exist!");
auto to_vertex = vertex_acc.find(delta.edge_create_delete.to_vertex);
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("The to vertex doesn't exist!");
auto from_lgo_vertex =
vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}});
if (from_lgo_vertex == vertex_acc.end()) {
throw RecoveryFailure("The from vertex doesn't exist!");
}
auto to_lgo_vertex = vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}});
if (to_lgo_vertex == vertex_acc.end()) {
throw RecoveryFailure("The to vertex doesn't exist!");
}
auto &from_vertex = from_lgo_vertex->vertex;
auto &to_vertex = to_lgo_vertex->vertex;
auto edge_gid = delta.edge_create_delete.gid;
auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
@ -716,16 +732,16 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
edge_ref = EdgeRef(&*edge);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*to_vertex, edge_ref};
auto it = std::find(from_vertex->out_edges.begin(), from_vertex->out_edges.end(), link);
if (it != from_vertex->out_edges.end()) throw RecoveryFailure("The from vertex already has this edge!");
from_vertex->out_edges.push_back(link);
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &to_vertex, edge_ref};
auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link);
if (it != from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex already has this edge!");
from_vertex.out_edges.push_back(link);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*from_vertex, edge_ref};
auto it = std::find(to_vertex->in_edges.begin(), to_vertex->in_edges.end(), link);
if (it != to_vertex->in_edges.end()) throw RecoveryFailure("The to vertex already has this edge!");
to_vertex->in_edges.push_back(link);
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &from_vertex, edge_ref};
auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link);
if (it != to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex already has this edge!");
to_vertex.in_edges.push_back(link);
}
ret.next_edge_id = std::max(ret.next_edge_id, edge_gid.AsUint() + 1);
@ -736,10 +752,17 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
break;
}
case WalDeltaData::Type::EDGE_DELETE: {
auto from_vertex = vertex_acc.find(delta.edge_create_delete.from_vertex);
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("The from vertex doesn't exist!");
auto to_vertex = vertex_acc.find(delta.edge_create_delete.to_vertex);
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("The to vertex doesn't exist!");
auto from_lgo_vertex =
vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.from_vertex.AsInt()}});
if (from_lgo_vertex == vertex_acc.end()) {
throw RecoveryFailure("The from vertex doesn't exist!");
}
auto to_lgo_vertex = vertex_acc.find(std::vector{PropertyValue{delta.edge_create_delete.to_vertex.AsInt()}});
if (to_lgo_vertex == vertex_acc.end()) {
throw RecoveryFailure("The to vertex doesn't exist!");
}
auto &from_vertex = from_lgo_vertex->vertex;
auto &to_vertex = to_lgo_vertex->vertex;
auto edge_gid = delta.edge_create_delete.gid;
auto edge_type_id = EdgeTypeId::FromUint(name_id_mapper->NameToId(delta.edge_create_delete.edge_type));
@ -750,18 +773,18 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
edge_ref = EdgeRef(&*edge);
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*to_vertex, edge_ref};
auto it = std::find(from_vertex->out_edges.begin(), from_vertex->out_edges.end(), link);
if (it == from_vertex->out_edges.end()) throw RecoveryFailure("The from vertex doesn't have this edge!");
std::swap(*it, from_vertex->out_edges.back());
from_vertex->out_edges.pop_back();
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &to_vertex, edge_ref};
auto it = std::find(from_vertex.out_edges.begin(), from_vertex.out_edges.end(), link);
if (it == from_vertex.out_edges.end()) throw RecoveryFailure("The from vertex doesn't have this edge!");
std::swap(*it, from_vertex.out_edges.back());
from_vertex.out_edges.pop_back();
}
{
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &*from_vertex, edge_ref};
auto it = std::find(to_vertex->in_edges.begin(), to_vertex->in_edges.end(), link);
if (it == to_vertex->in_edges.end()) throw RecoveryFailure("The to vertex doesn't have this edge!");
std::swap(*it, to_vertex->in_edges.back());
to_vertex->in_edges.pop_back();
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{edge_type_id, &from_vertex, edge_ref};
auto it = std::find(to_vertex.in_edges.begin(), to_vertex.in_edges.end(), link);
if (it == to_vertex.in_edges.end()) throw RecoveryFailure("The to vertex doesn't have this edge!");
std::swap(*it, to_vertex.in_edges.back());
to_vertex.in_edges.pop_back();
}
if (items.properties_on_edges) {
if (!edge_acc.remove(edge_gid)) throw RecoveryFailure("The edge must be removed here!");

3
src/storage/v3/durability/wal.hpp
View File

@ -25,6 +25,7 @@
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/skip_list.hpp"
@ -189,7 +190,7 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Storage
/// Function used to load the WAL data into the storage.
/// @throw RecoveryFailure
RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConstraints *indices_constraints,
std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
std::optional<uint64_t> last_loaded_timestamp, VerticesSkipList *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
Config::Items items);

10
src/storage/v3/indices.cpp
View File

@ -270,7 +270,7 @@ void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transacti
acc.insert(Entry{vertex, tx.start_timestamp});
}
bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices) {
bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
if (!emplaced) {
@ -279,7 +279,8 @@ bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor ve
}
try {
auto acc = it->second.access();
for (Vertex &vertex : vertices) {
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}
@ -416,7 +417,7 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
}
}
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices) {
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] =
index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
@ -426,7 +427,8 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::
}
try {
auto acc = it->second.access();
for (Vertex &vertex : vertices) {
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}

5
src/storage/v3/indices.hpp
View File

@ -19,6 +19,7 @@
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/bound.hpp"
#include "utils/logging.hpp"
#include "utils/skip_list.hpp"
@ -58,7 +59,7 @@ class LabelIndex {
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices);
bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
/// Returns false if there was no index to drop
bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
@ -156,7 +157,7 @@ class LabelPropertyIndex {
void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices);
bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }

40
src/storage/v3/key_store.cpp Normal file
View File

@ -0,0 +1,40 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <iterator>
#include <ranges>
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
KeyStore::KeyStore(const std::vector<PropertyValue> &key_values) {
for (auto i = 0; i < key_values.size(); ++i) {
MG_ASSERT(!key_values[i].IsNull());
store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
}
}
PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
std::vector<PropertyValue> KeyStore::Keys() const {
auto keys_map = store_.Properties();
std::vector<PropertyValue> keys;
keys.reserve(keys_map.size());
std::ranges::transform(
keys_map, std::back_inserter(keys),
[](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
return keys;
}
} // namespace memgraph::storage::v3

60
src/storage/v3/key_store.hpp Normal file
View File

@ -0,0 +1,60 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <algorithm>
#include <compare>
#include <functional>
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
class KeyStore {
public:
explicit KeyStore(const std::vector<PropertyValue> &key_values);
KeyStore(const KeyStore &) = delete;
KeyStore(KeyStore &&other) noexcept = default;
KeyStore &operator=(const KeyStore &) = delete;
KeyStore &operator=(KeyStore &&other) noexcept = default;
~KeyStore() = default;
PropertyValue GetKey(size_t index) const;
std::vector<PropertyValue> Keys() const;
friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
// TODO(antaljanosbenjamin): also compare the schema
return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
return std::ranges::equal(lhs.Keys(), rhs.Keys());
}
friend bool operator<(const KeyStore &lhs, const std::vector<PropertyValue> &rhs) {
// TODO(antaljanosbenjamin): also compare the schema
return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const std::vector<PropertyValue> &rhs) {
return std::ranges::equal(lhs.Keys(), rhs);
}
private:
PropertyStore store_;
};
} // namespace memgraph::storage::v3

42
src/storage/v3/lexicographically_ordered_vertex.hpp Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <concepts>
#include <type_traits>
#include "storage/v3/vertex.hpp"
#include "utils/concepts.hpp"
namespace memgraph::storage::v3 {
struct LexicographicallyOrderedVertex {
Vertex vertex;
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys == rhs.vertex.keys;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys < rhs.vertex.keys;
}
// TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys == rhs;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys < rhs;
}
};
} // namespace memgraph::storage::v3

7
src/storage/v3/replication/replication_server.cpp
View File

@ -167,9 +167,10 @@ void Storage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::B
LabelPropertyIndex(&storage_->indices_, &storage_->constraints_, storage_->config_.items);
try {
spdlog::debug("Loading snapshot");
auto recovered_snapshot = durability::LoadSnapshot(*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_,
&storage_->epoch_history_, &storage_->name_id_mapper_,
&storage_->edge_count_, storage_->config_.items);
auto recovered_snapshot = durability::RecoveredSnapshot{};
durability::LoadSnapshot(*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->epoch_history_,
&storage_->name_id_mapper_, &storage_->edge_count_, storage_->config_.items);
spdlog::debug("Snapshot loaded successfully");
// If this step is present it should always be the first step of
// the recovery so we use the UUID we read from snasphost

58
src/storage/v3/storage.cpp
View File

@ -29,6 +29,7 @@
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/replication/config.hpp"
#include "storage/v3/replication/replication_client.hpp"
#include "storage/v3/replication/replication_server.hpp"
@ -52,11 +53,11 @@ namespace {
inline constexpr uint16_t kEpochHistoryRetention = 1000;
} // namespace
auto AdvanceToVisibleVertex(utils::SkipList<Vertex>::Iterator it, utils::SkipList<Vertex>::Iterator end,
auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Iterator end,
std::optional<VertexAccessor> *vertex, Transaction *tx, View view, Indices *indices,
Constraints *constraints, Config::Items config) {
while (it != end) {
*vertex = VertexAccessor::Create(&*it, tx, indices, constraints, config, view);
*vertex = VertexAccessor::Create(&it->vertex, tx, indices, constraints, config, view);
if (!*vertex) {
++it;
continue;
@ -66,7 +67,7 @@ auto AdvanceToVisibleVertex(utils::SkipList<Vertex>::Iterator it, utils::SkipLis
return it;
}
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, utils::SkipList<Vertex>::Iterator it)
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it)
: self_(self),
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), &self->vertex_, self->transaction_, self->view_,
self->indices_, self_->constraints_, self->config_)) {}
@ -342,9 +343,11 @@ Storage::Storage(Config config)
config_.durability.storage_directory);
}
if (config_.durability.recover_on_startup) {
auto info = durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch_id_, &epoch_history_,
&vertices_, &edges_, &edge_count_, &name_id_mapper_, &indices_, &constraints_,
config_.items, &wal_seq_num_);
auto info = std::optional<durability::RecoveryInfo>{};
durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch_id_, &epoch_history_, &vertices_,
&edges_, &edge_count_, &name_id_mapper_, &indices_, &constraints_, config_.items,
&wal_seq_num_);
if (info) {
vertex_id_ = info->next_vertex_id;
edge_id_ = info->next_edge_id;
@ -467,11 +470,12 @@ VertexAccessor Storage::Accessor::CreateVertex() {
auto gid = storage_->vertex_id_.fetch_add(1, std::memory_order_acq_rel);
auto acc = storage_->vertices_.access();
auto *delta = CreateDeleteObjectDelta(&transaction_);
auto [it, inserted] = acc.insert(Vertex{Gid::FromUint(gid), delta});
// TODO(antaljanosbenjamin): handle keys and schema
auto [it, inserted] = acc.insert({Vertex{Gid::FromUint(gid), delta}});
MG_ASSERT(inserted, "The vertex must be inserted here!");
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
delta->prev.Set(&*it);
return {&*it, &transaction_, &storage_->indices_, &storage_->constraints_, config_};
delta->prev.Set(&it->vertex);
return {&it->vertex, &transaction_, &storage_->indices_, &storage_->constraints_, config_};
}
VertexAccessor Storage::Accessor::CreateVertex(Gid gid) {
@ -486,18 +490,20 @@ VertexAccessor Storage::Accessor::CreateVertex(Gid gid) {
std::memory_order_release);
auto acc = storage_->vertices_.access();
auto *delta = CreateDeleteObjectDelta(&transaction_);
auto [it, inserted] = acc.insert(Vertex{gid, delta});
auto [it, inserted] = acc.insert({Vertex{gid, delta}});
MG_ASSERT(inserted, "The vertex must be inserted here!");
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
delta->prev.Set(&*it);
return {&*it, &transaction_, &storage_->indices_, &storage_->constraints_, config_};
delta->prev.Set(&it->vertex);
return {&it->vertex, &transaction_, &storage_->indices_, &storage_->constraints_, config_};
}
std::optional<VertexAccessor> Storage::Accessor::FindVertex(Gid gid, View view) {
auto acc = storage_->vertices_.access();
auto it = acc.find(gid);
auto it = acc.find(std::vector{PropertyValue{gid.AsInt()}});
if (it == acc.end()) return std::nullopt;
return VertexAccessor::Create(&*it, &transaction_, &storage_->indices_, &storage_->constraints_, config_, view);
return VertexAccessor::Create(&it->vertex, &transaction_, &storage_->indices_, &storage_->constraints_, config_,
view);
return {};
}
Result<std::optional<VertexAccessor>> Storage::Accessor::DeleteVertex(VertexAccessor *vertex) {
@ -609,10 +615,10 @@ Result<EdgeAccessor> Storage::Accessor::CreateEdge(VertexAccessor *from, VertexA
// Obtain the locks by `gid` order to avoid lock cycles.
std::unique_lock<utils::SpinLock> guard_from(from_vertex->lock, std::defer_lock);
std::unique_lock<utils::SpinLock> guard_to(to_vertex->lock, std::defer_lock);
if (from_vertex->gid < to_vertex->gid) {
if (from_vertex < to_vertex) {
guard_from.lock();
guard_to.lock();
} else if (from_vertex->gid > to_vertex->gid) {
} else if (from_vertex > to_vertex) {
guard_to.lock();
guard_from.lock();
} else {
@ -669,10 +675,10 @@ Result<EdgeAccessor> Storage::Accessor::CreateEdge(VertexAccessor *from, VertexA
// Obtain the locks by `gid` order to avoid lock cycles.
std::unique_lock<utils::SpinLock> guard_from(from_vertex->lock, std::defer_lock);
std::unique_lock<utils::SpinLock> guard_to(to_vertex->lock, std::defer_lock);
if (from_vertex->gid < to_vertex->gid) {
if (&from_vertex < &to_vertex) {
guard_from.lock();
guard_to.lock();
} else if (from_vertex->gid > to_vertex->gid) {
} else if (&from_vertex > &to_vertex) {
guard_to.lock();
guard_from.lock();
} else {
@ -744,10 +750,10 @@ Result<std::optional<EdgeAccessor>> Storage::Accessor::DeleteEdge(EdgeAccessor *
// Obtain the locks by `gid` order to avoid lock cycles.
std::unique_lock<utils::SpinLock> guard_from(from_vertex->lock, std::defer_lock);
std::unique_lock<utils::SpinLock> guard_to(to_vertex->lock, std::defer_lock);
if (from_vertex->gid < to_vertex->gid) {
if (&from_vertex < &to_vertex) {
guard_from.lock();
guard_to.lock();
} else if (from_vertex->gid > to_vertex->gid) {
} else if (&from_vertex > &to_vertex) {
guard_to.lock();
guard_from.lock();
} else {
@ -1021,7 +1027,7 @@ void Storage::Accessor::Abort() {
}
case Delta::Action::DELETE_OBJECT: {
vertex->deleted = true;
my_deleted_vertices.push_back(vertex->gid);
my_deleted_vertices.push_back(vertex->Gid());
break;
}
case Delta::Action::RECREATE_OBJECT: {
@ -1412,7 +1418,7 @@ void Storage::CollectGarbage() {
}
vertex->delta = nullptr;
if (vertex->deleted) {
current_deleted_vertices.push_back(vertex->gid);
current_deleted_vertices.push_back(vertex->Gid());
}
break;
}
@ -1530,13 +1536,17 @@ void Storage::CollectGarbage() {
if constexpr (force) {
// if force is set to true, then we have unique_lock and no transactions are active
// so we can clean all of the deleted vertices
std::vector<PropertyValue> key(1);
while (!garbage_vertices_.empty()) {
MG_ASSERT(vertex_acc.remove(garbage_vertices_.front().second), "Invalid database state!");
key.front() = PropertyValue{garbage_vertices_.front().second.AsInt()};
MG_ASSERT(vertex_acc.remove(key), "Invalid database state!");
garbage_vertices_.pop_front();
}
} else {
std::vector<PropertyValue> key(1);
while (!garbage_vertices_.empty() && garbage_vertices_.front().first < oldest_active_start_timestamp) {
MG_ASSERT(vertex_acc.remove(garbage_vertices_.front().second), "Invalid database state!");
key.front() = PropertyValue{garbage_vertices_.front().second.AsInt()};
MG_ASSERT(vertex_acc.remove(key), "Invalid database state!");
garbage_vertices_.pop_front();
}
}

12
src/storage/v3/storage.hpp
View File

@ -27,12 +27,14 @@
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/file_locker.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/rw_lock.hpp"
@ -60,7 +62,7 @@ namespace memgraph::storage::v3 {
/// An instance of this will be usually be wrapped inside VerticesIterable for
/// generic, public use.
class AllVerticesIterable final {
utils::SkipList<Vertex>::Accessor vertices_accessor_;
VerticesSkipList::Accessor vertices_accessor_;
Transaction *transaction_;
View view_;
Indices *indices_;
@ -71,10 +73,10 @@ class AllVerticesIterable final {
public:
class Iterator final {
AllVerticesIterable *self_;
utils::SkipList<Vertex>::Iterator it_;
VerticesSkipList::Iterator it_;
public:
Iterator(AllVerticesIterable *self, utils::SkipList<Vertex>::Iterator it);
Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it);
VertexAccessor operator*() const;
@ -85,7 +87,7 @@ class AllVerticesIterable final {
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
AllVerticesIterable(utils::SkipList<Vertex>::Accessor vertices_accessor, Transaction *transaction, View view,
AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view,
Indices *indices, Constraints *constraints, Config::Items config)
: vertices_accessor_(std::move(vertices_accessor)),
transaction_(transaction),
@ -482,7 +484,7 @@ class Storage final {
mutable utils::RWLock main_lock_{utils::RWLock::Priority::WRITE};
// Main object storage
utils::SkipList<Vertex> vertices_;
VerticesSkipList vertices_;
utils::SkipList<Edge> edges_;
std::atomic<uint64_t> vertex_id_{0};
std::atomic<uint64_t> edge_id_{0};

13
src/storage/v3/vertex.hpp
View File

@ -13,24 +13,28 @@
#include <limits>
#include <tuple>
#include <type_traits>
#include <vector>
#include "storage/v3/delta.hpp"
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
#include "utils/spin_lock.hpp"
namespace memgraph::storage::v3 {
struct Vertex {
Vertex(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {
Vertex(Gid gid, Delta *delta) : keys{{PropertyValue{gid.AsInt()}}}, deleted(false), delta(delta) {
MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
"Vertex must be created with an initial DELETE_OBJECT delta!");
}
Gid gid;
Gid Gid() const { return Gid::FromInt(keys.GetKey(0).ValueInt()); }
KeyStore keys;
std::vector<LabelId> labels;
PropertyStore properties;
@ -47,9 +51,4 @@ struct Vertex {
static_assert(alignof(Vertex) >= 8, "The Vertex should be aligned to at least 8!");
inline bool operator==(const Vertex &first, const Vertex &second) { return first.gid == second.gid; }
inline bool operator<(const Vertex &first, const Vertex &second) { return first.gid < second.gid; }
inline bool operator==(const Vertex &first, const Gid &second) { return first.gid == second; }
inline bool operator<(const Vertex &first, const Gid &second) { return first.gid < second; }
} // namespace memgraph::storage::v3

5
src/storage/v3/vertex_accessor.hpp
View File

@ -94,7 +94,10 @@ class VertexAccessor final {
Result<size_t> OutDegree(View view) const;
Gid Gid() const noexcept { return vertex_->gid; }
Gid Gid() const noexcept {
// TODO(antaljanosbenjamin): remove this whole function.
return vertex_->Gid();
}
bool operator==(const VertexAccessor &other) const noexcept {
return vertex_ == other.vertex_ && transaction_ == other.transaction_;

19
src/storage/v3/vertices_skip_list.hpp Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::storage::v3 {
using VerticesSkipList = utils::SkipList<LexicographicallyOrderedVertex>;
} // namespace memgraph::storage::v3

16
src/utils/concepts.hpp
View File

@ -11,6 +11,7 @@
#pragma once
#include <concepts>
#include <iterator>
namespace memgraph::utils {
template <typename T, typename... Args>
@ -18,4 +19,19 @@ concept SameAsAnyOf = (std::same_as<T, Args> || ...);
template <typename T>
concept Enum = std::is_enum_v<T>;
// WithRef, CanReference and Dereferenceable is based on the similarly named concepts in GCC 11.2.0
// bits/iterator_concepts.h
template <typename T>
using WithRef = T &;
template <typename T>
concept CanReference = requires {
typename WithRef<T>;
};
template <typename T>
concept Dereferenceable = requires(T t) {
{ *t } -> CanReference;
};
} // namespace memgraph::utils

12
tests/unit/CMakeLists.txt
View File

@ -363,5 +363,15 @@ add_unit_test(websocket.cpp)
target_link_libraries(${test_prefix}websocket mg-communication Boost::headers)
# Test storage-v3
# Test utilities
add_library(storage_v3_test_utils storage_v3_test_utils.cpp)
target_link_libraries(storage_v3_test_utils mg-storage-v3)
add_unit_test(storage_v3.cpp)
target_link_libraries(${test_prefix}storage_v3 mg-storage-v3)
target_link_libraries(${test_prefix}storage_v3 mg-storage-v3 storage_v3_test_utils)
add_unit_test(storage_v3_property_store.cpp)
target_link_libraries(${test_prefix}storage_v3_property_store mg-storage-v3 fmt)
add_unit_test(storage_v3_key_store.cpp)
target_link_libraries(${test_prefix}storage_v3_key_store mg-storage-v3 rapidcheck rapidcheck_gtest)

2548
tests/unit/storage_v3.cpp
View File

File diff suppressed because it is too large Load Diff

46
tests/unit/storage_v3_key_store.cpp Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <string>
#include <vector>
/**
* gtest/gtest.h must be included before rapidcheck/gtest.h!
*/
#include <gtest/gtest.h>
#include <rapidcheck.h>
#include <rapidcheck/gtest.h>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3::test {
RC_GTEST_PROP(KeyStore, KeyStore, (std::vector<std::string> values)) {
RC_PRE(!values.empty());
std::vector<PropertyValue> property_values;
property_values.reserve(values.size());
std::transform(values.begin(), values.end(), std::back_inserter(property_values),
[](std::string &value) { return PropertyValue{std::move(value)}; });
KeyStore key_store{property_values};
const auto keys = key_store.Keys();
RC_ASSERT(keys.size() == property_values.size());
for (int i = 0; i < keys.size(); ++i) {
RC_ASSERT(keys[i] == property_values[i]);
RC_ASSERT(key_store.GetKey(i) == property_values[i]);
}
}
} // namespace memgraph::storage::v3::test

622
tests/unit/storage_v3_property_store.cpp Normal file
View File

@ -0,0 +1,622 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <array>
#include <limits>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::storage::v3::tests {
class StorageV3PropertyStore : public ::testing::Test {
protected:
PropertyStore props;
const std::array<PropertyValue, 24> kSampleValues = {
PropertyValue(),
PropertyValue(false),
PropertyValue(true),
PropertyValue(0),
PropertyValue(33),
PropertyValue(-33),
PropertyValue(-3137),
PropertyValue(3137),
PropertyValue(310000007),
PropertyValue(-310000007),
PropertyValue(3100000000007L),
PropertyValue(-3100000000007L),
PropertyValue(0.0),
PropertyValue(33.33),
PropertyValue(-33.33),
PropertyValue(3137.3137),
PropertyValue(-3137.3137),
PropertyValue("sample"),
PropertyValue(std::string(404, 'n')),
PropertyValue(
std::vector<PropertyValue>{PropertyValue(33), PropertyValue(std::string("sample")), PropertyValue(-33.33)}),
PropertyValue(std::vector<PropertyValue>{PropertyValue(), PropertyValue(false)}),
PropertyValue(std::map<std::string, PropertyValue>{{"sample", PropertyValue()}, {"key", PropertyValue(false)}}),
PropertyValue(std::map<std::string, PropertyValue>{
{"test", PropertyValue(33)}, {"map", PropertyValue(std::string("sample"))}, {"item", PropertyValue(-33.33)}}),
PropertyValue(TemporalData(TemporalType::Date, 23)),
};
void AssertPropertyIsEqual(const PropertyStore &store, PropertyId property, const PropertyValue &value) {
ASSERT_TRUE(store.IsPropertyEqual(property, value));
for (const auto &sample : kSampleValues) {
if (sample == value) {
ASSERT_TRUE(store.IsPropertyEqual(property, sample));
} else {
ASSERT_FALSE(store.IsPropertyEqual(property, sample));
}
}
}
};
using testing::UnorderedElementsAre;
TEST_F(StorageV3PropertyStore, StoreTwoProperties) {
const auto make_prop = [](int64_t prop_id_and_value) {
auto prop = PropertyId::FromInt(prop_id_and_value);
auto value = PropertyValue(prop_id_and_value);
return std::make_pair(prop, value);
};
const auto first_prop_and_value = make_prop(42);
const auto second_prop_and_value = make_prop(43);
ASSERT_TRUE(props.SetProperty(first_prop_and_value.first, first_prop_and_value.second));
ASSERT_TRUE(props.SetProperty(second_prop_and_value.first, second_prop_and_value.second));
ASSERT_THAT(props.Properties(), UnorderedElementsAre(first_prop_and_value, second_prop_and_value));
}
TEST_F(StorageV3PropertyStore, Simple) {
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, SimpleLarge) {
auto prop = PropertyId::FromInt(42);
{
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
{
auto value = PropertyValue(TemporalData(TemporalType::Date, 23));
ASSERT_FALSE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
ASSERT_FALSE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptySetToNull) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, Clear) {
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props.SetProperty(prop, value));
ASSERT_EQ(props.GetProperty(prop), value);
ASSERT_TRUE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, value);
ASSERT_THAT(props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_TRUE(props.ClearProperties());
ASSERT_TRUE(props.GetProperty(prop).IsNull());
ASSERT_FALSE(props.HasProperty(prop));
AssertPropertyIsEqual(props, prop, PropertyValue());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptyClear) {
ASSERT_FALSE(props.ClearProperties());
ASSERT_EQ(props.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveConstruct) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
PropertyStore props2(std::move(props1));
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveConstructLarge) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
PropertyStore props2(std::move(props1));
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveAssign) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(42);
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
auto value2 = PropertyValue(68);
PropertyStore props2;
ASSERT_TRUE(props2.SetProperty(prop, value2));
ASSERT_EQ(props2.GetProperty(prop), value2);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value2);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value2)));
props2 = std::move(props1);
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, MoveAssignLarge) {
PropertyStore props1;
auto prop = PropertyId::FromInt(42);
auto value = PropertyValue(std::string(10000, 'a'));
ASSERT_TRUE(props1.SetProperty(prop, value));
ASSERT_EQ(props1.GetProperty(prop), value);
ASSERT_TRUE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, value);
ASSERT_THAT(props1.Properties(), UnorderedElementsAre(std::pair(prop, value)));
{
auto value2 = PropertyValue(std::string(10000, 'b'));
PropertyStore props2;
ASSERT_TRUE(props2.SetProperty(prop, value2));
ASSERT_EQ(props2.GetProperty(prop), value2);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value2);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value2)));
props2 = std::move(props1);
ASSERT_EQ(props2.GetProperty(prop), value);
ASSERT_TRUE(props2.HasProperty(prop));
AssertPropertyIsEqual(props2, prop, value);
ASSERT_THAT(props2.Properties(), UnorderedElementsAre(std::pair(prop, value)));
}
// NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move,hicpp-invalid-access-moved)
ASSERT_TRUE(props1.GetProperty(prop).IsNull());
ASSERT_FALSE(props1.HasProperty(prop));
AssertPropertyIsEqual(props1, prop, PropertyValue());
ASSERT_EQ(props1.Properties().size(), 0);
}
TEST_F(StorageV3PropertyStore, EmptySet) {
std::vector<PropertyValue> vec{PropertyValue(true), PropertyValue(123), PropertyValue()};
std::map<std::string, PropertyValue> map{{"nandare", PropertyValue(false)}};
const TemporalData temporal{TemporalType::LocalDateTime, 23};
std::vector<PropertyValue> data{PropertyValue(true), PropertyValue(123), PropertyValue(123.5),
PropertyValue("nandare"), PropertyValue(vec), PropertyValue(map),
PropertyValue(temporal)};
auto prop = PropertyId::FromInt(42);
for (const auto &value : data) {
PropertyStore local_props;
ASSERT_TRUE(local_props.SetProperty(prop, value));
ASSERT_EQ(local_props.GetProperty(prop), value);
ASSERT_TRUE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, value);
ASSERT_THAT(local_props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(local_props.SetProperty(prop, value));
ASSERT_EQ(local_props.GetProperty(prop), value);
ASSERT_TRUE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, value);
ASSERT_THAT(local_props.Properties(), UnorderedElementsAre(std::pair(prop, value)));
ASSERT_FALSE(local_props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(local_props.GetProperty(prop).IsNull());
ASSERT_FALSE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, PropertyValue());
ASSERT_EQ(local_props.Properties().size(), 0);
ASSERT_TRUE(local_props.SetProperty(prop, PropertyValue()));
ASSERT_TRUE(local_props.GetProperty(prop).IsNull());
ASSERT_FALSE(local_props.HasProperty(prop));
AssertPropertyIsEqual(local_props, prop, PropertyValue());
ASSERT_EQ(local_props.Properties().size(), 0);
}
}
TEST_F(StorageV3PropertyStore, FullSet) {
std::vector<PropertyValue> vec{PropertyValue(true), PropertyValue(123), PropertyValue()};
std::map<std::string, PropertyValue> map{{"nandare", PropertyValue(false)}};
const TemporalData temporal{TemporalType::LocalDateTime, 23};
std::map<PropertyId, PropertyValue> data{
{PropertyId::FromInt(1), PropertyValue(true)}, {PropertyId::FromInt(2), PropertyValue(123)},
{PropertyId::FromInt(3), PropertyValue(123.5)}, {PropertyId::FromInt(4), PropertyValue("nandare")},
{PropertyId::FromInt(5), PropertyValue(vec)}, {PropertyId::FromInt(6), PropertyValue(map)},
{PropertyId::FromInt(7), PropertyValue(temporal)}};
std::vector<PropertyValue> alt{PropertyValue(),
PropertyValue(std::string()),
PropertyValue(std::string(10, 'a')),
PropertyValue(std::string(100, 'a')),
PropertyValue(std::string(1000, 'a')),
PropertyValue(std::string(10000, 'a')),
PropertyValue(std::string(100000, 'a'))};
for (const auto &target : data) {
for (const auto &item : data) {
ASSERT_TRUE(props.SetProperty(item.first, item.second));
}
for (size_t i = 0; i < alt.size(); ++i) {
if (i == 1) {
ASSERT_TRUE(props.SetProperty(target.first, alt[i]));
} else {
ASSERT_FALSE(props.SetProperty(target.first, alt[i]));
}
for (const auto &item : data) {
if (item.first == target.first) {
ASSERT_EQ(props.GetProperty(item.first), alt[i]);
if (alt[i].IsNull()) {
ASSERT_FALSE(props.HasProperty(item.first));
} else {
ASSERT_TRUE(props.HasProperty(item.first));
}
AssertPropertyIsEqual(props, item.first, alt[i]);
} else {
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
}
auto current = data;
if (alt[i].IsNull()) {
current.erase(target.first);
} else {
current[target.first] = alt[i];
}
ASSERT_EQ(props.Properties(), current);
}
for (ssize_t i = alt.size() - 1; i >= 0; --i) {
ASSERT_FALSE(props.SetProperty(target.first, alt[i]));
for (const auto &item : data) {
if (item.first == target.first) {
ASSERT_EQ(props.GetProperty(item.first), alt[i]);
if (alt[i].IsNull()) {
ASSERT_FALSE(props.HasProperty(item.first));
} else {
ASSERT_TRUE(props.HasProperty(item.first));
}
AssertPropertyIsEqual(props, item.first, alt[i]);
} else {
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
}
auto current = data;
if (alt[i].IsNull()) {
current.erase(target.first);
} else {
current[target.first] = alt[i];
}
ASSERT_EQ(props.Properties(), current);
}
ASSERT_TRUE(props.SetProperty(target.first, target.second));
ASSERT_EQ(props.GetProperty(target.first), target.second);
ASSERT_TRUE(props.HasProperty(target.first));
AssertPropertyIsEqual(props, target.first, target.second);
props.ClearProperties();
ASSERT_EQ(props.Properties().size(), 0);
for (const auto &item : data) {
ASSERT_TRUE(props.GetProperty(item.first).IsNull());
ASSERT_FALSE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, PropertyValue());
}
}
}
TEST_F(StorageV3PropertyStore, IntEncoding) {
std::map<PropertyId, PropertyValue> data{
// {PropertyId::FromUint(0UL),
// PropertyValue(std::numeric_limits<int64_t>::min())},
// {PropertyId::FromUint(10UL), PropertyValue(-137438953472L)},
// {PropertyId::FromUint(std::numeric_limits<uint8_t>::max()),
// PropertyValue(-4294967297L)},
// {PropertyId::FromUint(256UL),
// PropertyValue(std::numeric_limits<int32_t>::min())},
// {PropertyId::FromUint(1024UL), PropertyValue(-1048576L)},
// {PropertyId::FromUint(1025UL), PropertyValue(-65537L)},
// {PropertyId::FromUint(1026UL),
// PropertyValue(std::numeric_limits<int16_t>::min())},
// {PropertyId::FromUint(1027UL), PropertyValue(-1024L)},
// {PropertyId::FromUint(2000UL), PropertyValue(-257L)},
// {PropertyId::FromUint(3000UL),
// PropertyValue(std::numeric_limits<int8_t>::min())},
// {PropertyId::FromUint(4000UL), PropertyValue(-1L)},
// {PropertyId::FromUint(10000UL), PropertyValue(0L)},
// {PropertyId::FromUint(20000UL), PropertyValue(1L)},
// {PropertyId::FromUint(30000UL),
// PropertyValue(std::numeric_limits<int8_t>::max())},
// {PropertyId::FromUint(40000UL), PropertyValue(256L)},
// {PropertyId::FromUint(50000UL), PropertyValue(1024L)},
// {PropertyId::FromUint(std::numeric_limits<uint16_t>::max()),
// PropertyValue(std::numeric_limits<int16_t>::max())},
// {PropertyId::FromUint(65536UL), PropertyValue(65536L)},
// {PropertyId::FromUint(1048576UL), PropertyValue(1048576L)},
// {PropertyId::FromUint(std::numeric_limits<uint32_t>::max()),
// PropertyValue(std::numeric_limits<int32_t>::max())},
{PropertyId::FromUint(4294967296UL), PropertyValue(4294967296L)},
{PropertyId::FromUint(137438953472UL), PropertyValue(137438953472L)},
{PropertyId::FromUint(std::numeric_limits<uint64_t>::max()), PropertyValue(std::numeric_limits<int64_t>::max())}};
for (const auto &item : data) {
ASSERT_TRUE(props.SetProperty(item.first, item.second));
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
for (auto it = data.rbegin(); it != data.rend(); ++it) {
const auto &item = *it;
ASSERT_FALSE(props.SetProperty(item.first, item.second)) << item.first.AsInt();
ASSERT_EQ(props.GetProperty(item.first), item.second);
ASSERT_TRUE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, item.second);
}
ASSERT_EQ(props.Properties(), data);
props.ClearProperties();
ASSERT_EQ(props.Properties().size(), 0);
for (const auto &item : data) {
ASSERT_TRUE(props.GetProperty(item.first).IsNull());
ASSERT_FALSE(props.HasProperty(item.first));
AssertPropertyIsEqual(props, item.first, PropertyValue());
}
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualIntAndDouble) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(42)));
std::vector<std::pair<PropertyValue, PropertyValue>> tests{
{PropertyValue(0), PropertyValue(0.0)},
{PropertyValue(123), PropertyValue(123.0)},
{PropertyValue(12345), PropertyValue(12345.0)},
{PropertyValue(12345678), PropertyValue(12345678.0)},
{PropertyValue(1234567890123L), PropertyValue(1234567890123.0)},
};
// Test equality with raw values.
for (auto test : tests) {
ASSERT_EQ(test.first, test.second);
// Test first, second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test second, first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
// Make both negative
test.first = PropertyValue(test.first.ValueInt() * -1);
test.second = PropertyValue(test.second.ValueDouble() * -1.0);
ASSERT_EQ(test.first, test.second);
// Test -first, -second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test -second, -first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
}
// Test equality with values wrapped in lists.
for (auto test : tests) {
test.first = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.first.ValueInt())});
test.second = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.second.ValueDouble())});
ASSERT_EQ(test.first, test.second);
// Test first, second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test second, first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
// Make both negative
test.first = PropertyValue(std::vector<PropertyValue>{PropertyValue(test.first.ValueList()[0].ValueInt() * -1)});
test.second =
PropertyValue(std::vector<PropertyValue>{PropertyValue(test.second.ValueList()[0].ValueDouble() * -1.0)});
ASSERT_EQ(test.first, test.second);
// Test -first, -second
ASSERT_FALSE(props.SetProperty(prop, test.first));
ASSERT_EQ(props.GetProperty(prop), test.first);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
// Test -second, -first
ASSERT_FALSE(props.SetProperty(prop, test.second));
ASSERT_EQ(props.GetProperty(prop), test.second);
ASSERT_TRUE(props.HasProperty(prop));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.second));
ASSERT_TRUE(props.IsPropertyEqual(prop, test.first));
}
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualString) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue("test")));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue("test")));
// Different length.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("helloworld")));
// Same length, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("asdf")));
// Shortened and extended.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("tes")));
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue("testt")));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualList) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(
props.SetProperty(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test")})));
ASSERT_TRUE(
props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test")})));
// Different length.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(24)})));
// Same length, different value.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("asdf")})));
// Shortened and extended.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42)})));
ASSERT_FALSE(props.IsPropertyEqual(
prop, PropertyValue(std::vector<PropertyValue>{PropertyValue(42), PropertyValue("test"), PropertyValue(true)})));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualMap) {
auto prop = PropertyId::FromInt(42);
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("test")}})));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("test")}})));
// Different length.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{{"fgh", PropertyValue(24)}})));
// Same length, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyx", PropertyValue("testt")}})));
// Same length, different key (different length).
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyxw", PropertyValue("test")}})));
// Same length, different key (same length).
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"zyw", PropertyValue("test")}})));
// Shortened and extended.
ASSERT_FALSE(
props.IsPropertyEqual(prop, PropertyValue(std::map<std::string, PropertyValue>{{"abc", PropertyValue(42)}})));
ASSERT_FALSE(props.IsPropertyEqual(
prop, PropertyValue(std::map<std::string, PropertyValue>{
{"abc", PropertyValue(42)}, {"sdf", PropertyValue(true)}, {"zyx", PropertyValue("test")}})));
}
TEST_F(StorageV3PropertyStore, IsPropertyEqualTemporalData) {
auto prop = PropertyId::FromInt(42);
const TemporalData temporal{TemporalType::Date, 23};
ASSERT_TRUE(props.SetProperty(prop, PropertyValue(temporal)));
ASSERT_TRUE(props.IsPropertyEqual(prop, PropertyValue(temporal)));
// Different type.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(TemporalData{TemporalType::Duration, 23})));
// Same type, different value.
ASSERT_FALSE(props.IsPropertyEqual(prop, PropertyValue(TemporalData{TemporalType::Date, 30})));
}
} // namespace memgraph::storage::v3::tests

23
tests/unit/storage_v3_test_utils.cpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage_v3_test_utils.hpp"
namespace memgraph::storage::v3::tests {
size_t CountVertices(Storage::Accessor &storage_accessor, View view) {
auto vertices = storage_accessor.Vertices(view);
size_t count = 0U;
for (auto it = vertices.begin(); it != vertices.end(); ++it, ++count)
;
return count;
}
} // namespace memgraph::storage::v3::tests

21
tests/unit/storage_v3_test_utils.hpp Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/storage.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3::tests {
size_t CountVertices(Storage::Accessor &storage_accessor, View view);
} // namespace memgraph::storage::v3::tests