tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Edge serializaiton

Browse Source
This commit is contained in:
Andi Skrgat 2023-04-05 15:29:44 +02:00
parent b8d956294b
commit f070329e77
7 changed files with 344 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/query/db_accessor.hpp
View File

@ -63,6 +63,10 @@ class EdgeAccessor final {
auto Properties(storage::View view) const { return impl_.Properties(view); } auto Properties(storage::View view) const { return impl_.Properties(view); }
auto PropertyStore() const { return impl_.PropertyStore(); }
void SetPropertyStore(const std::string_view buffer) const { impl_.SetPropertyStore(buffer); }
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const { storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
return impl_.GetProperty(key, view); return impl_.GetProperty(key, view);
} }
@ -93,6 +97,8 @@ class EdgeAccessor final {
storage::Gid Gid() const noexcept { return impl_.Gid(); } storage::Gid Gid() const noexcept { return impl_.Gid(); }
void SetGid(storage::Gid gid_) { impl_.SetGid(gid_); }
bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; } bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; }
bool operator!=(const EdgeAccessor &e) const noexcept { return !(*this == e); } bool operator!=(const EdgeAccessor &e) const noexcept { return !(*this == e); }
@ -121,9 +127,9 @@ class VertexAccessor final {
auto Properties(storage::View view) const { return impl_.Properties(view); } auto Properties(storage::View view) const { return impl_.Properties(view); }
auto PropertyStore(storage::View view) const { return impl_.PropertyStore(view); } auto PropertyStore() const { return impl_.PropertyStore(); }
void SetPropertyStore(const std::string_view buffer) { impl_.SetPropertyStore(buffer); } void SetPropertyStore(const std::string_view buffer) const { impl_.SetPropertyStore(buffer); }
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const { storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
return impl_.GetProperty(key, view); return impl_.GetProperty(key, view);

176
src/storage/rocks/storage.hpp
View File

@ -19,12 +19,14 @@
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <tuple>
#include "query/common.hpp" #include "query/common.hpp"
#include "query/db_accessor.hpp" #include "query/db_accessor.hpp"
#include "spdlog/spdlog.h" #include "spdlog/spdlog.h"
#include "storage/rocks/loopback.hpp" #include "storage/rocks/loopback.hpp"
#include "storage/rocks/serialization.hpp" #include "storage/rocks/serialization.hpp"
#include "storage/v2/delta.hpp" #include "storage/v2/delta.hpp"
#include "storage/v2/edge.hpp"
#include "storage/v2/id_types.hpp" #include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp" #include "storage/v2/property_value.hpp"
#include "storage/v2/storage.hpp" #include "storage/v2/storage.hpp"
@ -60,18 +62,38 @@ class RocksDBStorage {
delete db_; delete db_;
} }
/* // STORING PART
Serialize and store in-memory vertex to the disk. // -----------------------------------------------------------
*/
// Serialize and store in-memory vertex to the disk.
// TODO: write the exact format
// Properties are serialized as the value
bool StoreVertex(const query::VertexAccessor &vertex_acc) { bool StoreVertex(const query::VertexAccessor &vertex_acc) {
const std::string key = SerializeVertex(vertex_acc); auto status =
rocksdb::Status status = db_->Put(rocksdb::WriteOptions(), key, "properties"); db_->Put(rocksdb::WriteOptions(), SerializeVertex(vertex_acc), SerializeProperties(vertex_acc.PropertyStore()));
// TODO: we need a better status check
return status.ok(); return status.ok();
} }
/* // TODO: remove config being sent as the parameter. Later will be added as the part of the storage accessor as in the
Clear all entries from the database. // memory version. for now assume that we always operate with edges having properties
*/ bool StoreEdge(const query::EdgeAccessor &edge_acc) {
// This can be improved with IIFE initialization lambda concept
std::string src_dest_key;
std::string dest_src_key;
std::tie(src_dest_key, dest_src_key) = SerializeEdge(edge_acc);
const std::string value = SerializeProperties(edge_acc.PropertyStore());
// const std::string value;
auto src_dest_status = db_->Put(rocksdb::WriteOptions(), src_dest_key, value);
auto dest_src_status = db_->Put(rocksdb::WriteOptions(), dest_src_key, value);
// TODO: improve a status check
return src_dest_status.ok() && dest_src_status.ok();
}
// UPDATE PART
// -----------------------------------------------------------
// Clear all entries from the database.
void Clear() { void Clear() {
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
@ -79,41 +101,41 @@ class RocksDBStorage {
} }
} }
/* // READ PART
Read all vertices stored in the database. // -----------------------------------------------------------
*/
std::vector<std::unique_ptr<query::VertexAccessor>> Vertices(query::DbAccessor &dba) { // TODO: change it to std::optional if the value doesn't exist
std::vector<std::unique_ptr<query::VertexAccessor>> vertices; // TODO: if the need comes for using also a GID object, use std::variant
query::VertexAccessor Vertex(const std::string_view gid, query::DbAccessor &dba) {
std::string value;
auto status = db_->Get(rocksdb::ReadOptions(), gid, &value);
MG_ASSERT(status.ok());
return DeserializeVertex(gid, value, dba);
}
// Read all vertices stored in the database.
// TODO: remove the semantics of unique pointers
std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba) {
std::vector<query::VertexAccessor> vertices;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); std::string key = it->key().ToString();
std::string value = it->value().ToString(); std::string value = it->value().ToString();
spdlog::debug("Key: {} Value: {}", key, value); spdlog::debug("Key: {} Value: {}", key, value);
auto res = DeserializeVertex(key, dba); vertices.push_back(DeserializeVertex(key, value, dba));
vertices.push_back(std::move(res));
} }
return vertices; return vertices;
} }
protected: // This method is used only for testing. Receives explicit key from which the edge accessor should be loaded
// std::string SerializeProperties(const auto &&properties) { query::EdgeAccessor LoadEdge(const std::string_view key, query::DbAccessor &dba) {
// if (properties.HasError() || (*properties).empty()) { std::string value;
// return ""; auto status = db_->Get(rocksdb::ReadOptions(), key, &value);
// } MG_ASSERT(status.ok());
// const auto &properties_iter = (*properties).begin(); return DeserializeEdge(key, value, dba);
// auto SerializeEntry = [](const auto &property_entry) { }
// std::stringstream prop_value;
// prop_value << property_entry.second; // PropertyValue has an overload for serialization
// return std::to_string(property_entry.first.AsUint()) + ":" + prop_value.str();
// };
// std::string result = SerializeEntry(*properties_iter);
// std::string ser_props = std::accumulate(std::next((*properties).begin()), (*properties).end(), result,
// [&SerializeEntry](const std::string &join, const auto &prop_entry) {
// return join + "," + SerializeEntry(prop_entry);
// });
// return ser_props;
// }
protected:
std::string SerializeProperties(const auto &&properties) { std::string SerializeProperties(const auto &&properties) {
if (properties.HasError()) { if (properties.HasError()) {
return ""; return "";
@ -132,24 +154,21 @@ class RocksDBStorage {
return ser_labels; return ser_labels;
} }
std::string SerializeGid(const storage::Gid &gid) { return std::to_string(gid.AsUint()); } // TODO: write a documentation for the method
// TODO: add deserialization equivalent method
inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); }
std::string SerializeVertex(const query::VertexAccessor &vertex_acc) { std::string SerializeVertex(const query::VertexAccessor &vertex_acc) {
// don't put before serialize labels delimiter // don't put before serialize labels delimiter
// don't put at the end delimiter
// Serialize labels
std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|"; std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|";
// Serialize gid result += SerializeIdType(vertex_acc.Gid()) + "|";
result += SerializeGid(vertex_acc.Gid()) + "|";
// result += SerializeProperties(vertex_acc.Properties(storage::View::OLD));
result += SerializeProperties(vertex_acc.PropertyStore(storage::View::OLD));
spdlog::info("Serialized vertex: {}", result);
return result; return result;
} }
std::unique_ptr<query::VertexAccessor> DeserializeVertex(const std::string_view key, query::DbAccessor &dba) { query::VertexAccessor DeserializeVertex(const std::string_view key, const std::string_view value,
query::DbAccessor &dba) {
// Create vertex // Create vertex
auto impl = std::make_unique<query::VertexAccessor>(dba.InsertVertex()); auto impl = query::VertexAccessor(dba.InsertVertex());
spdlog::info("Key to deserialize: {}", key); spdlog::info("Key to deserialize: {}", key);
const auto vertex_parts = utils::Split(key, "|"); const auto vertex_parts = utils::Split(key, "|");
// Deserialize labels // Deserialize labels
@ -157,7 +176,7 @@ class RocksDBStorage {
const auto labels = utils::Split(vertex_parts[0], ","); const auto labels = utils::Split(vertex_parts[0], ",");
for (const auto &label : labels) { for (const auto &label : labels) {
const storage::LabelId label_id = storage::LabelId::FromUint(std::stoull(label)); const storage::LabelId label_id = storage::LabelId::FromUint(std::stoull(label));
auto maybe_error = impl->AddLabel(label_id); auto maybe_error = impl.AddLabel(label_id);
if (maybe_error.HasError()) { if (maybe_error.HasError()) {
switch (maybe_error.GetError()) { switch (maybe_error.GetError()) {
case storage::Error::SERIALIZATION_ERROR: case storage::Error::SERIALIZATION_ERROR:
@ -173,24 +192,65 @@ class RocksDBStorage {
} }
} }
// deserialize gid // deserialize gid
const storage::Gid gid = storage::Gid::FromUint(std::stoull(vertex_parts[1])); impl.SetGid(storage::Gid::FromUint(std::stoull(vertex_parts[1])));
impl->SetGid(gid);
// deserialize properties // deserialize properties
impl->SetPropertyStore(vertex_parts[2]); impl.SetPropertyStore(value);
// const auto ser_properties = utils::Split(vertex_parts[2], ",");
// std::map<storage::PropertyId, storage::PropertyValue> vec_properties;
// for (const auto &prop_entry : ser_properties) {
// const auto &split_prop_entry = utils::Split(prop_entry, ":");
// // this is a problem since the value will always contain a string
// vec_properties.emplace(storage::PropertyId::FromUint(std::stoull(split_prop_entry[0])),
// storage::PropertyValue(split_prop_entry[1]));
// }
// query::MultiPropsInitChecked(impl.get(), vec_properties);
return impl; return impl;
} }
// Serializes edge accessor to obtain a key for the key-value store
// returns two string because there will be two keys since edge is stored in both directions
std::pair<std::string, std::string> SerializeEdge(const query::EdgeAccessor &edge_acc) {
// Serialized objects
auto from_gid = SerializeIdType(edge_acc.From().Gid());
auto to_gid = SerializeIdType(edge_acc.To().Gid());
auto edge_type = SerializeIdType(edge_acc.EdgeType());
auto edge_gid = SerializeIdType(edge_acc.Gid());
// source->destination key
std::string src_dest_key = from_gid + "|";
src_dest_key += to_gid + "|";
src_dest_key += "0|";
src_dest_key += edge_type + "|";
src_dest_key += edge_gid + "|";
// destination->source key
std::string dest_src_key = to_gid + "|";
dest_src_key += from_gid + "|";
dest_src_key += "1|";
dest_src_key += edge_type + "|";
dest_src_key += edge_gid + "|";
return {src_dest_key, dest_src_key};
}
// deserialize edge from the given key-value
query::EdgeAccessor DeserializeEdge(const std::string_view key, const std::string_view value,
query::DbAccessor &dba) {
// first you need to deserialize vertices
const auto edge_parts = utils::Split(key, "|");
std::string from_gid;
std::string to_gid;
// TODO: Use IIFE to load
if (edge_parts[2] == "0") { // out edge
from_gid = edge_parts[0];
to_gid = edge_parts[1];
} else { // in edge
from_gid = edge_parts[1];
to_gid = edge_parts[0];
}
// load vertex accessors
auto from_acc = Vertex(from_gid, dba);
auto to_acc = Vertex(to_gid, dba);
// TODO: remove to deserialization edge type id method
const auto edge_type_id = storage::EdgeTypeId::FromUint(std::stoull(edge_parts[3]));
// TODO: remove to deserialization edge type id method
const auto edge_gid = storage::Gid::FromUint(std::stoull(edge_parts[4]));
const auto maybe_edge = dba.InsertEdge(&from_acc, &to_acc, edge_type_id);
MG_ASSERT(maybe_edge.HasValue());
auto edge_impl = query::EdgeAccessor(*maybe_edge);
// in the new storage API, setting gid must be done atomically
edge_impl.SetGid(edge_gid);
return edge_impl;
}
private: private:
// rocksdb internals // rocksdb internals
rocksdb::Options options_; rocksdb::Options options_;

11
src/storage/v2/edge_accessor.cpp
View File

@ -214,7 +214,8 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view)
properties = edge_.ptr->properties.Properties(); properties = edge_.ptr->properties.Properties();
delta = edge_.ptr->delta; delta = edge_.ptr->delta;
} }
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) { return std::move(properties);
/*ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
switch (delta.action) { switch (delta.action) {
case Delta::Action::SET_PROPERTY: { case Delta::Action::SET_PROPERTY: {
auto it = properties.find(delta.property.key); auto it = properties.find(delta.property.key);
@ -251,6 +252,14 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view)
if (!exists) return Error::NONEXISTENT_OBJECT; if (!exists) return Error::NONEXISTENT_OBJECT;
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT; if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
return std::move(properties); return std::move(properties);
*/
} }
Result<std::string> EdgeAccessor::PropertyStore() const {
// TODO: needs changing where there are no properties on edges
return edge_.ptr->properties.StringBuffer();
}
void EdgeAccessor::SetPropertyStore(std::string_view buffer) const { edge_.ptr->properties.SetBuffer(buffer); }
} // namespace memgraph::storage } // namespace memgraph::storage

12
src/storage/v2/edge_accessor.hpp
View File

@ -73,6 +73,10 @@ class EdgeAccessor final {
/// @throw std::bad_alloc /// @throw std::bad_alloc
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const; Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Result<std::string> PropertyStore() const;
void SetPropertyStore(std::string_view buffer) const;
Gid Gid() const noexcept { Gid Gid() const noexcept {
if (config_.properties_on_edges) { if (config_.properties_on_edges) {
return edge_.ptr->gid; return edge_.ptr->gid;
@ -81,6 +85,14 @@ class EdgeAccessor final {
} }
} }
void SetGid(storage::Gid gid_) {
if (config_.properties_on_edges) {
edge_.ptr->gid = storage::Gid::FromUint(gid_.AsUint());
} else {
edge_.gid = storage::Gid::FromUint(gid_.AsUint());
}
}
bool IsCycle() const { return from_vertex_ == to_vertex_; } bool IsCycle() const { return from_vertex_ == to_vertex_; }
bool operator==(const EdgeAccessor &other) const noexcept { bool operator==(const EdgeAccessor &other) const noexcept {

4
src/storage/v2/vertex_accessor.cpp
View File

@ -308,9 +308,9 @@ Result<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View view
return std::move(value); return std::move(value);
} }
Result<std::string> VertexAccessor::PropertyStore(View /*unused*/) const { return vertex_->properties.StringBuffer(); } Result<std::string> VertexAccessor::PropertyStore() const { return vertex_->properties.StringBuffer(); }
void VertexAccessor::SetPropertyStore(const std::string_view buffer) { vertex_->properties.SetBuffer(buffer); } void VertexAccessor::SetPropertyStore(const std::string_view buffer) const { vertex_->properties.SetBuffer(buffer); }
Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view) const { Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view) const {
bool exists = true; bool exists = true;

4
src/storage/v2/vertex_accessor.hpp
View File

@ -86,9 +86,9 @@ class VertexAccessor final {
/// @throw std::bad_alloc /// @throw std::bad_alloc
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const; Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Result<std::string> PropertyStore(View view) const; Result<std::string> PropertyStore() const;
void SetPropertyStore(const std::string_view buffer); void SetPropertyStore(std::string_view buffer) const;
/// @throw std::bad_alloc /// @throw std::bad_alloc
/// @throw std::length_error if the resulting vector exceeds /// @throw std::length_error if the resulting vector exceeds

345
tests/unit/storage_rocks.cpp
View File

@ -24,174 +24,213 @@
#include "storage/v2/vertex_accessor.hpp" #include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/view.hpp" #include "storage/v2/view.hpp"
class RocksDBStorageTest : public ::testing::TestWithParam<bool> {}; class RocksDBStorageTest : public ::testing::TestWithParam<bool> {
public:
TEST(RocksDBStorageTest, SerializeVertexGID) {
memgraph::storage::rocks::RocksDBStorage db; memgraph::storage::rocks::RocksDBStorage db;
try { ~RocksDBStorageTest() { db.Clear(); }
memgraph::storage::Storage storage; };
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba); TEST_F(RocksDBStorageTest, SerializeVertexGID) {
std::unordered_set<uint64_t> gids; memgraph::storage::Storage storage;
for (uint64_t i = 0; i < 5; ++i) { auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
gids.insert(i); memgraph::query::DbAccessor dba(&storage_dba);
auto impl = dba.InsertVertex(); std::unordered_set<uint64_t> gids;
impl.SetGid(memgraph::storage::Gid::FromUint(i)); for (uint64_t i = 0; i < 5; ++i) {
db.StoreVertex(impl); gids.insert(i);
} auto impl = dba.InsertVertex();
// load vertices from disk impl.SetGid(memgraph::storage::Gid::FromUint(i));
auto loaded_vertices = db.Vertices(dba); db.StoreVertex(impl);
db.Clear(); }
ASSERT_EQ(loaded_vertices.size(), 5); // load vertices from disk
for (const auto &vertex_acc : loaded_vertices) { auto loaded_vertices = db.Vertices(dba);
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint())); ASSERT_EQ(loaded_vertices.size(), 5);
} for (const auto &vertex_acc : loaded_vertices) {
} catch (std::exception &) { ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
db.Clear();
} }
} }
// TODO: clean labels string // TODO: clean labels string
TEST(RocksDBStorageTest, SerializeVertexGIDLabels) { TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
memgraph::storage::rocks::RocksDBStorage db; memgraph::storage::Storage storage;
try { auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::storage::Storage storage; memgraph::query::DbAccessor dba(&storage_dba);
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); // save vertices on disk
memgraph::query::DbAccessor dba(&storage_dba); std::vector<std::string> labels{"Player", "Person", "Ball"};
// save vertices on disk std::unordered_set<uint64_t> gids;
std::vector<std::string> labels{"Player", "Person", "Ball"}; std::vector<memgraph::storage::LabelId> label_ids;
std::unordered_set<uint64_t> gids; for (int i = 0; i < 3; i++) {
std::vector<memgraph::storage::LabelId> label_ids; label_ids.push_back(dba.NameToLabel(labels[i]));
for (int i = 0; i < 3; i++) { }
label_ids.push_back(dba.NameToLabel(labels[i])); for (int i = 0; i < 5; ++i) {
} gids.insert(i);
auto impl = dba.InsertVertex();
for (int i = 0; i < 5; ++i) { impl.SetGid(memgraph::storage::Gid::FromUint(i));
gids.insert(i); impl.AddLabel(dba.NameToLabel(labels[i % 3]));
auto impl = dba.InsertVertex(); db.StoreVertex(impl);
impl.SetGid(memgraph::storage::Gid::FromUint(i)); }
impl.AddLabel(dba.NameToLabel(labels[i % 3])); // load vertices from disk
db.StoreVertex(impl); auto loaded_vertices = db.Vertices(dba);
} ASSERT_EQ(loaded_vertices.size(), 5);
// load vertices from disk for (const auto &vertex_acc : loaded_vertices) {
auto loaded_vertices = db.Vertices(dba); ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
db.Clear(); auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(loaded_vertices.size(), 5); ASSERT_EQ(labels->size(), 1);
for (const auto &vertex_acc : loaded_vertices) { ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint())); return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD); }));
ASSERT_EQ(labels->size(), 1);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
} catch (std::exception &) {
db.Clear();
} }
} }
// TODO: clean labels string // TODO: clean labels string
TEST(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) { TEST_F(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) {
memgraph::storage::rocks::RocksDBStorage db; memgraph::storage::Storage storage;
try { auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::storage::Storage storage; memgraph::query::DbAccessor dba(&storage_dba);
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); // save vertices on disk
memgraph::query::DbAccessor dba(&storage_dba); std::vector<std::string> labels{"Player", "Person", "Ball"};
// save vertices on disk std::unordered_set<uint64_t> gids;
std::vector<std::string> labels{"Player", "Person", "Ball"}; std::vector<memgraph::storage::LabelId> label_ids;
std::unordered_set<uint64_t> gids; for (int i = 0; i < 3; i++) {
std::vector<memgraph::storage::LabelId> label_ids; label_ids.push_back(dba.NameToLabel(labels[i]));
for (int i = 0; i < 3; i++) { }
label_ids.push_back(dba.NameToLabel(labels[i])); for (int i = 0; i < 5; ++i) {
} gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
}
for (int i = 0; i < 5; ++i) { TEST_F(RocksDBStorageTest, SerializeVertexGIDProperties) {
gids.insert(i); memgraph::storage::Storage storage;
auto impl = dba.InsertVertex(); auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
impl.SetGid(memgraph::storage::Gid::FromUint(i)); memgraph::query::DbAccessor dba(&storage_dba);
impl.AddLabel(label_ids[i % 3]); // prepare labels
impl.AddLabel(label_ids[(i + 1) % 3]); std::vector<std::string> labels{"Player", "Person", "Ball"};
db.StoreVertex(impl); std::vector<memgraph::storage::LabelId> label_ids;
} for (int i = 0; i < 3; i++) {
// load vertices from disk label_ids.push_back(dba.NameToLabel(labels[i]));
auto loaded_vertices = db.Vertices(dba); }
db.Clear(); // prepare properties
ASSERT_EQ(loaded_vertices.size(), 5); std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
for (const auto &vertex_acc : loaded_vertices) { properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint())); properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD); properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
// gids
std::unordered_set<uint64_t> gids;
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
memgraph::query::MultiPropsInitChecked(&impl, properties);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// labels
{
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2); ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) { ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end(); return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
})); }));
} }
} catch (std::exception &e) { {
db.Clear(); // check properties
auto props = vertex_acc.Properties(memgraph::storage::View::OLD);
ASSERT_FALSE(props.HasError());
auto prop_name = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name"));
auto prop_memory = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory"));
auto prop_price = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price"));
auto prop_unexisting = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random"));
ASSERT_TRUE(prop_name->IsString());
ASSERT_EQ(prop_name->ValueString(), "disk");
ASSERT_TRUE(prop_memory->IsString());
ASSERT_EQ(prop_memory->ValueString(), "1TB");
// TODO: needs to be solved
ASSERT_TRUE(prop_price->IsDouble());
ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21);
ASSERT_TRUE(prop_unexisting->IsNull());
}
} }
} }
TEST(RocksDBStorageTest, SerializeVertexGIDProperties) { // TEST_F(RocksDBStorageTest, SerializeEdgeNewTest) {
memgraph::storage::rocks::RocksDBStorage db; // memgraph::storage::Storage storage;
try { // auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::storage::Storage storage; // memgraph::query::DbAccessor dba(&storage_dba);
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); // std::vector<std::string> labels{"Player", "Person", "Ball"};
memgraph::query::DbAccessor dba(&storage_dba); // std::unordered_set<uint64_t> gids;
// prepare labels // std::vector<memgraph::storage::LabelId> label_ids;
std::vector<std::string> labels{"Player", "Person", "Ball"}; // for (int i = 0; i < 3; i++) {
std::vector<memgraph::storage::LabelId> label_ids; // label_ids.push_back(dba.NameToLabel(labels[i]));
for (int i = 0; i < 3; i++) { // }
label_ids.push_back(dba.NameToLabel(labels[i])); // for (int i = 0; i < 5; ++i) {
} // gids.insert(i);
// prepare properties // auto impl = dba.InsertVertex();
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties; // impl.SetGid(memgraph::storage::Gid::FromUint(i));
properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk")); // impl.AddLabel(label_ids[i % 3]);
properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB")); // impl.AddLabel(label_ids[(i + 1) % 3]);
properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21)); // db.StoreVertex(impl);
// gids // }
std::unordered_set<uint64_t> gids; // spdlog::info("Finished...");
// ASSERT_TRUE(true);
for (int i = 0; i < 5; ++i) { // memgraph::storage::Storage storage;
gids.insert(i); // auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
auto impl = dba.InsertVertex(); // memgraph::query::DbAccessor dba(&storage_dba);
impl.SetGid(memgraph::storage::Gid::FromUint(i)); // prepare labels
impl.AddLabel(label_ids[i % 3]); // std::vector<std::string> labels{"Player", "Person"};
impl.AddLabel(label_ids[(i + 1) % 3]); // prepare vertex properties
memgraph::query::MultiPropsInitChecked(&impl, properties); // std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> vertex_properties;
db.StoreVertex(impl); // vertex_properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
} // vertex_properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
// load vertices from disk // vertex_properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
auto loaded_vertices = db.Vertices(dba); // create first vertex
db.Clear(); // uint64_t src_id = 0;
ASSERT_EQ(loaded_vertices.size(), 5); // auto impl_src = dba.InsertVertex();
for (const auto &vertex_acc : loaded_vertices) { // impl_src.SetGid(memgraph::storage::Gid::FromUint(src_id));
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint())); // impl_src.AddLabel(dba.NameToLabel(labels[src_id]));
// labels // memgraph::query::MultiPropsInitChecked(&impl_src, vertex_properties);
{ // db.StoreVertex(impl_src);
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD); // create second vertex
ASSERT_EQ(labels->size(), 2); // uint64_t dest_id = 1;
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) { // auto impl_dest = dba.InsertVertex();
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end(); // impl_dest.SetGid(memgraph::storage::Gid::FromUint(dest_id));
})); // impl_dest.AddLabel(dba.NameToLabel(labels[dest_id]));
} // memgraph::query::MultiPropsInitChecked(&impl_dest, vertex_properties);
{ // db.StoreVertex(impl_dest);
// check properties // // prepare edge properties
auto props = vertex_acc->Properties(memgraph::storage::View::OLD); // std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> edge_properties;
ASSERT_FALSE(props.HasError()); // edge_properties.emplace(dba.NameToProperty("sum"), memgraph::storage::PropertyValue("2TB"));
auto prop_name = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name")); // edge_properties.emplace(dba.NameToProperty("same_type"), memgraph::storage::PropertyValue(true));
auto prop_memory = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory")); // edge_properties.emplace(dba.NameToProperty("cluster_price"), memgraph::storage::PropertyValue(2000.42));
auto prop_price = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price")); // create an edge between them
auto prop_unexisting = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random")); // uint64_t edge_gid = 2;
ASSERT_TRUE(prop_name->IsString()); // auto edge_type_id = "CONNECTION";
ASSERT_EQ(prop_name->ValueString(), "disk"); // auto impl_edge = dba.InsertEdge(&impl_src, &impl_dest, dba.NameToEdgeType(edge_type_id));
ASSERT_TRUE(prop_memory->IsString()); // ASSERT_FALSE(impl_edge.HasError());
ASSERT_EQ(prop_memory->ValueString(), "1TB"); // (*impl_edge).SetGid(memgraph::storage::Gid::FromUint(edge_gid));
// TODO: needs to be solved // memgraph::query::MultiPropsInitChecked(&*impl_edge, edge_properties);
ASSERT_TRUE(prop_price->IsDouble()); // // load vertices from disk
ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21); // auto loaded_vertices = db.Vertices(dba);
ASSERT_TRUE(prop_unexisting->IsNull()); // ASSERT_EQ(loaded_vertices.size(), 2);
} // ASSERT_FALSE(true);
} // }
} catch (std::exception &e) {
db.Clear();
}
}