Code cleaning

This commit is contained in:
Andi Skrgat 2023-04-06 16:28:29 +02:00
parent 2019f5cc72
commit 1bed62da4a
6 changed files with 206 additions and 263 deletions

View File

@ -30,6 +30,7 @@
#include "storage/v2/edge_accessor.hpp" #include "storage/v2/edge_accessor.hpp"
#include "storage/v2/id_types.hpp" #include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp" #include "storage/v2/property_value.hpp"
#include "storage/v2/result.hpp"
#include "storage/v2/storage.hpp" #include "storage/v2/storage.hpp"
#include "storage/v2/vertex.hpp" #include "storage/v2/vertex.hpp"
#include "storage/v2/vertex_accessor.hpp" #include "storage/v2/vertex_accessor.hpp"
@ -41,25 +42,26 @@
namespace memgraph::storage::rocks { namespace memgraph::storage::rocks {
inline void CheckRocksDBStatus(rocksdb::Status status) { MG_ASSERT(status.ok(), "rocksdb: {}", status.ToString()); }
class RocksDBStorage { class RocksDBStorage {
public: public:
explicit RocksDBStorage() { explicit RocksDBStorage() {
options_.create_if_missing = true; options_.create_if_missing = true;
// options_.OptimizeLevelStyleCompaction(); // options_.OptimizeLevelStyleCompaction();
std::filesystem::path rocksdb_path = "./rocks_experiment_unit_test"; std::filesystem::path rocksdb_path = "./rocks_experiment_unit_test";
if (!utils::EnsureDir(rocksdb_path)) { MG_ASSERT(utils::EnsureDir(rocksdb_path), "Unable to create storage folder on the disk.");
SPDLOG_ERROR("Unable to create storage folder on disk."); CheckRocksDBStatus(rocksdb::DB::Open(options_, rocksdb_path, &db_));
// TODO: throw some error CheckRocksDBStatus(db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "vertex", &vertex_chandle));
} CheckRocksDBStatus(db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "edge", &edge_chandle));
rocksdb::Status status = rocksdb::DB::Open(options_, rocksdb_path, &db_);
if (!status.ok()) {
spdlog::error(status.ToString());
}
} }
// TODO: explicitly delete other constructors
~RocksDBStorage() { ~RocksDBStorage() {
rocksdb::Status status = db_->Close(); CheckRocksDBStatus(db_->DropColumnFamily(vertex_chandle));
MG_ASSERT(status.ok()); CheckRocksDBStatus(db_->DropColumnFamily(edge_chandle));
CheckRocksDBStatus(db_->Close());
delete db_; delete db_;
} }
@ -69,37 +71,32 @@ class RocksDBStorage {
// Serialize and store in-memory vertex to the disk. // Serialize and store in-memory vertex to the disk.
// TODO: write the exact format // TODO: write the exact format
// Properties are serialized as the value // Properties are serialized as the value
bool StoreVertex(const query::VertexAccessor &vertex_acc) { void StoreVertex(const query::VertexAccessor &vertex_acc) {
auto status = CheckRocksDBStatus(db_->Put(rocksdb::WriteOptions(), vertex_chandle, SerializeVertex(vertex_acc),
db_->Put(rocksdb::WriteOptions(), SerializeVertex(vertex_acc), SerializeProperties(vertex_acc.PropertyStore())); SerializeProperties(vertex_acc.PropertyStore())));
// TODO: we need a better status check
return status.ok();
} }
// TODO: remove config being sent as the parameter. Later will be added as the part of the storage accessor as in the // TODO: remove config being sent as the parameter. Later will be added as the part of the storage accessor as in the
// memory version. for now assume that we always operate with edges having properties // memory version. for now assume that we always operate with edges having properties
bool StoreEdge(const query::EdgeAccessor &edge_acc) { void StoreEdge(const query::EdgeAccessor &edge_acc) {
// This can be improved with IIFE initialization lambda concept auto [src_dest_key, dest_src_key] = SerializeEdge(edge_acc);
std::string src_dest_key;
std::string dest_src_key;
std::tie(src_dest_key, dest_src_key) = SerializeEdge(edge_acc);
const std::string value = SerializeProperties(edge_acc.PropertyStore()); const std::string value = SerializeProperties(edge_acc.PropertyStore());
// const std::string value; CheckRocksDBStatus(db_->Put(rocksdb::WriteOptions(), edge_chandle, src_dest_key, value));
auto src_dest_status = db_->Put(rocksdb::WriteOptions(), src_dest_key, value); CheckRocksDBStatus(db_->Put(rocksdb::WriteOptions(), edge_chandle, dest_src_key, value));
auto dest_src_status = db_->Put(rocksdb::WriteOptions(), dest_src_key, value);
// TODO: improve a status check
return src_dest_status.ok() && dest_src_status.ok();
} }
// UPDATE PART // UPDATE PART
// ----------------------------------------------------------- // -----------------------------------------------------------
// Clear all entries from the database. // Clear all entries from the database.
// TODO: check if this deletes all entries, or you also need to specify handle here
// TODO: This will not be needed in the production code and can possibly removed in testing
void Clear() { void Clear() {
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
db_->Delete(rocksdb::WriteOptions(), it->key().ToString()); db_->Delete(rocksdb::WriteOptions(), it->key().ToString());
} }
delete it;
} }
// READ PART // READ PART
@ -110,14 +107,14 @@ class RocksDBStorage {
// This should be part of edge accessor since it should be impossible to search vertex by a gid // This should be part of edge accessor since it should be impossible to search vertex by a gid
// This should again be changed when we have mulitple same vertices // This should again be changed when we have mulitple same vertices
std::optional<query::VertexAccessor> Vertex(const std::string_view gid, query::DbAccessor &dba) { std::optional<query::VertexAccessor> Vertex(const std::string_view gid, query::DbAccessor &dba) {
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
const std::string_view key = it->key().ToStringView(); const std::string_view key = it->key().ToStringView();
const auto entry_parts = utils::Split(key, "|"); if (key.starts_with(gid)) {
if (entry_parts.size() == 2 && entry_parts[0] == gid) {
return DeserializeVertex(key, it->value().ToStringView(), dba); return DeserializeVertex(key, it->value().ToStringView(), dba);
} }
} }
delete it;
return std::nullopt; return std::nullopt;
} }
@ -128,46 +125,33 @@ class RocksDBStorage {
// we use the firt way since this should be possible to optimize using Bloom filters and prefix search // we use the firt way since this should be possible to optimize using Bloom filters and prefix search
std::vector<query::EdgeAccessor> OutEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) { std::vector<query::EdgeAccessor> OutEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) {
std::vector<query::EdgeAccessor> out_edges; std::vector<query::EdgeAccessor> out_edges;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), edge_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); const std::string_view key = it->key().ToStringView();
const auto vertex_parts = utils::Split(key, "|"); const auto vertex_parts = utils::Split(key, "|");
// When logical partitioning will be introduced we will know exactly in which logical partition are we searching if (vertex_parts[0] == SerializeIdType(vertex_acc.Gid()) && vertex_parts[2] == "0") {
// But nothing should break now also since we are checking number of vertex parts out_edges.push_back(DeserializeEdge(key, it->value().ToStringView(), dba));
// if there are more than two vertex parts, it must be an edge entry
// ATM check one would suffice, even without checking number of vertex parts since as the first entry in vertex
// serialization, vertex labels are being saved. This might change since this is the direct trade-off between
// finding fast vertices by labels and finding fast "the other vertex" from an edg
if (vertex_parts[0] == SerializeIdType(vertex_acc.Gid()) && vertex_parts.size() > 2 && vertex_parts[2] == "0") {
std::string value = it->value().ToString();
out_edges.push_back(DeserializeEdge(key, value, dba));
} }
} }
delete it;
return out_edges; return out_edges;
} }
// TODO: currently search in the same database instance vertices and edges but change later to column families // InEdges of one vertex_acc with GID "dest_gid" have following format in the RocksDB:
// In edges of one vertex_acc with gid dest_gid have following format in the RocksDB:
// other_vertex_gid | dest_gid | 0 | ... // other_vertex_gid | dest_gid | 0 | ...
// dest_gid | other_verte_gid | 1 | ... // dest_gid | other_verte_gid | 1 | ...
// we use the second way since this should be possible to optimize using Bloom filters and prefix search. // we use the second way since this should be possible to optimize using Bloom filters and prefix search.
std::vector<query::EdgeAccessor> InEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) { std::vector<query::EdgeAccessor> InEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) {
std::vector<query::EdgeAccessor> in_edges; std::vector<query::EdgeAccessor> in_edges;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), edge_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); const std::string_view key = it->key().ToStringView();
const auto vertex_parts = utils::Split(key, "|"); const auto vertex_parts = utils::Split(key, "|");
// When logical partitioning will be introduced we will know exactly in which logical partition are we searching if (vertex_parts[0] == SerializeIdType(vertex_acc.Gid()) && vertex_parts[2] == "1") {
// But nothing should break now also since we are checking number of vertex parts in_edges.push_back(DeserializeEdge(key, it->value().ToStringView(), dba));
// if there are more than two vertex parts, it must be an edge entry
// ATM check one would suffice, even without checking number of vertex parts since as the first entry in vertex
// serialization, vertex labels are being saved. This might change since this is the direct trade-off between
// finding fast vertices by labels and finding fast "the other vertex" from an edg
if (vertex_parts[0] == SerializeIdType(vertex_acc.Gid()) && vertex_parts.size() > 2 && vertex_parts[2] == "1") {
std::string value = it->value().ToString();
in_edges.push_back(DeserializeEdge(key, value, dba));
} }
} }
delete it;
return in_edges; return in_edges;
} }
@ -176,19 +160,14 @@ class RocksDBStorage {
// certainly can be a bit optimized so that new object isn't created if can be discarded // certainly can be a bit optimized so that new object isn't created if can be discarded
std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::LabelId &label_id) { std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::LabelId &label_id) {
std::vector<query::VertexAccessor> vertices; std::vector<query::VertexAccessor> vertices;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); auto vertex = DeserializeVertex(it->key().ToStringView(), it->value().ToStringView(), dba);
std::string value = it->value().ToString();
if (utils::Split(key, "|").size() > 2) {
continue;
}
spdlog::debug("Key: {} Value: {}", key, value);
auto vertex = DeserializeVertex(key, value, dba);
if (const auto res = vertex.HasLabel(storage::View::OLD, label_id); !res.HasError() && *res) { if (const auto res = vertex.HasLabel(storage::View::OLD, label_id); !res.HasError() && *res) {
vertices.push_back(vertex); vertices.push_back(vertex);
} }
} }
delete it;
return vertices; return vertices;
} }
@ -198,44 +177,30 @@ class RocksDBStorage {
std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::PropertyId &property_id, std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::PropertyId &property_id,
const storage::PropertyValue &prop_value) { const storage::PropertyValue &prop_value) {
std::vector<query::VertexAccessor> vertices; std::vector<query::VertexAccessor> vertices;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); auto vertex = DeserializeVertex(it->key().ToStringView(), it->value().ToStringView(), dba);
std::string value = it->value().ToString();
if (utils::Split(key, "|").size() > 2) {
continue;
}
auto vertex = DeserializeVertex(key, value, dba);
if (const auto res = vertex.GetProperty(storage::View::OLD, property_id); !res.HasError() && *res == prop_value) { if (const auto res = vertex.GetProperty(storage::View::OLD, property_id); !res.HasError() && *res == prop_value) {
vertices.push_back(vertex); vertices.push_back(vertex);
} }
} }
delete it;
return vertices; return vertices;
} }
// Read all vertices stored in the database. // Read all vertices stored in the database.
std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba) { std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba) {
std::vector<query::VertexAccessor> vertices; std::vector<query::VertexAccessor> vertices;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions()); rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle);
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
std::string key = it->key().ToString(); vertices.push_back(DeserializeVertex(it->key().ToStringView(), it->value().ToStringView(), dba));
std::string value = it->value().ToString();
if (utils::Split(key, "|").size() > 2) {
continue;
}
spdlog::debug("Key: {} Value: {}", key, value);
vertices.push_back(DeserializeVertex(key, value, dba));
} }
delete it;
return vertices; return vertices;
} }
protected: protected:
std::string SerializeProperties(const auto &&properties) { inline std::string SerializeProperties(const auto &&properties) { return properties; }
if (properties.HasError()) {
return "";
}
return *properties;
}
std::string SerializeLabels(const auto &&labels) { std::string SerializeLabels(const auto &&labels) {
if (labels.HasError() || (*labels).empty()) { if (labels.HasError() || (*labels).empty()) {
@ -253,7 +218,6 @@ class RocksDBStorage {
inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); } inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); }
std::string SerializeVertex(const query::VertexAccessor &vertex_acc) { std::string SerializeVertex(const query::VertexAccessor &vertex_acc) {
// don't put before serialize labels delimiter
std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|"; std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|";
result += SerializeIdType(vertex_acc.Gid()); result += SerializeIdType(vertex_acc.Gid());
return result; return result;
@ -318,18 +282,16 @@ class RocksDBStorage {
// deserialize edge from the given key-value // deserialize edge from the given key-value
query::EdgeAccessor DeserializeEdge(const std::string_view key, const std::string_view value, query::EdgeAccessor DeserializeEdge(const std::string_view key, const std::string_view value,
query::DbAccessor &dba) { query::DbAccessor &dba) {
// first you need to deserialize vertices
const auto edge_parts = utils::Split(key, "|"); const auto edge_parts = utils::Split(key, "|");
std::string from_gid; auto [from_gid, to_gid] = std::invoke(
std::string to_gid; [&](const auto &edge_parts) {
// TODO: Use IIFE to load if (edge_parts[2] == "0") { // out edge
if (edge_parts[2] == "0") { // out edge return std::make_pair(edge_parts[0], edge_parts[1]);
from_gid = edge_parts[0]; }
to_gid = edge_parts[1]; // in edge
} else { // in edge return std::make_pair(edge_parts[1], edge_parts[0]);
from_gid = edge_parts[1]; },
to_gid = edge_parts[0]; edge_parts);
}
// load vertex accessors // load vertex accessors
auto from_acc = Vertex(from_gid, dba); auto from_acc = Vertex(from_gid, dba);
auto to_acc = Vertex(to_gid, dba); auto to_acc = Vertex(to_gid, dba);
@ -350,12 +312,10 @@ class RocksDBStorage {
} }
private: private:
// rocksdb internals
rocksdb::Options options_; rocksdb::Options options_;
rocksdb::DB *db_; rocksdb::DB *db_;
// slk::Loopback loopback_; rocksdb::ColumnFamilyHandle *vertex_chandle = nullptr;
// slk::Encoder encoder_{loopback_.GetBuilder()}; rocksdb::ColumnFamilyHandle *edge_chandle = nullptr;
// slk::Decoder decoder_{loopback_.GetReader()};
}; };
} // namespace memgraph::storage::rocks } // namespace memgraph::storage::rocks

View File

@ -255,10 +255,7 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view)
*/ */
} }
Result<std::string> EdgeAccessor::PropertyStore() const { std::string EdgeAccessor::PropertyStore() const { return edge_.ptr->properties.StringBuffer(); }
// TODO: needs changing where there are no properties on edges
return edge_.ptr->properties.StringBuffer();
}
void EdgeAccessor::SetPropertyStore(std::string_view buffer) const { edge_.ptr->properties.SetBuffer(buffer); } void EdgeAccessor::SetPropertyStore(std::string_view buffer) const { edge_.ptr->properties.SetBuffer(buffer); }

View File

@ -73,7 +73,7 @@ class EdgeAccessor final {
/// @throw std::bad_alloc /// @throw std::bad_alloc
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const; Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Result<std::string> PropertyStore() const; std::string PropertyStore() const;
void SetPropertyStore(std::string_view buffer) const; void SetPropertyStore(std::string_view buffer) const;

View File

@ -308,7 +308,7 @@ Result<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View view
return std::move(value); return std::move(value);
} }
Result<std::string> VertexAccessor::PropertyStore() const { return vertex_->properties.StringBuffer(); } std::string VertexAccessor::PropertyStore() const { return vertex_->properties.StringBuffer(); }
void VertexAccessor::SetPropertyStore(const std::string_view buffer) const { vertex_->properties.SetBuffer(buffer); } void VertexAccessor::SetPropertyStore(const std::string_view buffer) const { vertex_->properties.SetBuffer(buffer); }

View File

@ -86,7 +86,7 @@ class VertexAccessor final {
/// @throw std::bad_alloc /// @throw std::bad_alloc
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const; Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Result<std::string> PropertyStore() const; std::string PropertyStore() const;
void SetPropertyStore(std::string_view buffer) const; void SetPropertyStore(std::string_view buffer) const;

View File

@ -21,180 +21,118 @@
#include "storage/v2/id_types.hpp" #include "storage/v2/id_types.hpp"
#include "storage/v2/isolation_level.hpp" #include "storage/v2/isolation_level.hpp"
#include "storage/v2/property_value.hpp" #include "storage/v2/property_value.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/vertex_accessor.hpp" #include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/view.hpp" #include "storage/v2/view.hpp"
class RocksDBStorageTest : public ::testing::TestWithParam<bool> { class RocksDBStorageTest : public ::testing::TestWithParam<bool> {
public: public:
memgraph::storage::rocks::RocksDBStorage db;
~RocksDBStorageTest() { db.Clear(); } ~RocksDBStorageTest() { db.Clear(); }
protected:
memgraph::storage::rocks::RocksDBStorage db;
memgraph::storage::Storage storage;
}; };
// // TEST_F(RocksDBStorageTest, SerializeVertexGID) { TEST_F(RocksDBStorageTest, SerializeVertexGID) {
// memgraph::storage::Storage storage; // empty vertices, only gid is serialized
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// memgraph::query::DbAccessor dba(&storage_dba);
// std::unordered_set<uint64_t> gids;
// for (uint64_t i = 0; i < 5; ++i) {
// gids.insert(i);
// auto impl = dba.InsertVertex();
// impl.SetGid(memgraph::storage::Gid::FromUint(i));
// db.StoreVertex(impl);
// }
// // load vertices from disk
// auto loaded_vertices = db.Vertices(dba);
// ASSERT_EQ(loaded_vertices.size(), 5);
// for (const auto &vertex_acc : loaded_vertices) {
// ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// }
// }
// TODO: clean labels string
// TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
// memgraph::storage::Storage storage;
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// memgraph::query::DbAccessor dba(&storage_dba);
// // save vertices on disk
// std::vector<std::string> labels{"Player", "Person", "Ball"};
// std::unordered_set<uint64_t> gids;
// std::vector<memgraph::storage::LabelId> label_ids;
// for (int i = 0; i < 3; i++) {
// label_ids.push_back(dba.NameToLabel(labels[i]));
// }
// for (int i = 0; i < 5; ++i) {
// gids.insert(i);
// auto impl = dba.InsertVertex();
// impl.SetGid(memgraph::storage::Gid::FromUint(i));
// impl.AddLabel(dba.NameToLabel(labels[i % 3]));
// db.StoreVertex(impl);
// }
// // load vertices from disk
// auto loaded_vertices = db.Vertices(dba);
// ASSERT_EQ(loaded_vertices.size(), 5);
// for (const auto &vertex_acc : loaded_vertices) {
// ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
// ASSERT_EQ(labels->size(), 1);
// ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
// return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
// }));
// }
// }
// TODO: clean labels string
// TEST_F(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) {
// memgraph::storage::Storage storage;
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// memgraph::query::DbAccessor dba(&storage_dba);
// // save vertices on disk
// std::vector<std::string> labels{"Player", "Person", "Ball"};
// std::unordered_set<uint64_t> gids;
// std::vector<memgraph::storage::LabelId> label_ids;
// for (int i = 0; i < 3; i++) {
// label_ids.push_back(dba.NameToLabel(labels[i]));
// }
// for (int i = 0; i < 5; ++i) {
// gids.insert(i);
// auto impl = dba.InsertVertex();
// impl.SetGid(memgraph::storage::Gid::FromUint(i));
// impl.AddLabel(label_ids[i % 3]);
// impl.AddLabel(label_ids[(i + 1) % 3]);
// db.StoreVertex(impl);
// }
// // load vertices from disk
// auto loaded_vertices = db.Vertices(dba);
// ASSERT_EQ(loaded_vertices.size(), 5);
// for (const auto &vertex_acc : loaded_vertices) {
// ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
// ASSERT_EQ(labels->size(), 2);
// ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
// return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
// }));
// }
// }
// TEST_F(RocksDBStorageTest, SerializeVertexGIDProperties) {
// memgraph::storage::Storage storage;
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// memgraph::query::DbAccessor dba(&storage_dba);
// // prepare labels
// std::vector<std::string> labels{"Player", "Person", "Ball"};
// std::vector<memgraph::storage::LabelId> label_ids;
// for (int i = 0; i < 3; i++) {
// label_ids.push_back(dba.NameToLabel(labels[i]));
// }
// // prepare properties
// std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
// properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
// properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
// properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
// // gids
// std::unordered_set<uint64_t> gids;
// for (int i = 0; i < 5; ++i) {
// gids.insert(i);
// auto impl = dba.InsertVertex();
// impl.SetGid(memgraph::storage::Gid::FromUint(i));
// impl.AddLabel(label_ids[i % 3]);
// impl.AddLabel(label_ids[(i + 1) % 3]);
// memgraph::query::MultiPropsInitChecked(&impl, properties);
// db.StoreVertex(impl);
// }
// // load vertices from disk
// auto loaded_vertices = db.Vertices(dba);
// ASSERT_EQ(loaded_vertices.size(), 5);
// for (const auto &vertex_acc : loaded_vertices) {
// ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// // labels
// {
// auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
// ASSERT_EQ(labels->size(), 2);
// ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
// return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
// }));
// }
// {
// // check properties
// auto props = vertex_acc.Properties(memgraph::storage::View::OLD);
// ASSERT_FALSE(props.HasError());
// auto prop_name = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name"));
// auto prop_memory = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory"));
// auto prop_price = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price"));
// auto prop_unexisting = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random"));
// ASSERT_TRUE(prop_name->IsString());
// ASSERT_EQ(prop_name->ValueString(), "disk");
// ASSERT_TRUE(prop_memory->IsString());
// ASSERT_EQ(prop_memory->ValueString(), "1TB");
// // TODO: needs to be solved
// ASSERT_TRUE(prop_price->IsDouble());
// ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21);
// ASSERT_TRUE(prop_unexisting->IsNull());
// }
// }
// }
TEST_F(RocksDBStorageTest, GetVerticesByLabel) {
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba); memgraph::query::DbAccessor dba(&storage_dba);
// prepare labels std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Player"), for (uint64_t i = 0; i < 5; ++i) {
dba.NameToLabel("Ball")}; gids.insert(i);
// insert vertices
for (int i = 0; i < 5; ++i) {
auto impl = dba.InsertVertex(); auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
}
}
// TODO: clean labels string
TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
// serialize vertex's gid with its single label
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
dba.NameToLabel("Ball")};
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]); impl.AddLabel(label_ids[i % 3]);
db.StoreVertex(impl); db.StoreVertex(impl);
} }
// load vertices from disk // load vertices from disk
auto player_vertices = db.Vertices(dba, dba.NameToLabel("Player")); auto loaded_vertices = db.Vertices(dba);
auto ball_vertices = db.Vertices(dba, dba.NameToLabel("Ball")); ASSERT_EQ(loaded_vertices.size(), 5);
ASSERT_EQ(player_vertices.size(), 4); for (const auto &vertex_acc : loaded_vertices) {
ASSERT_EQ(ball_vertices.size(), 1); ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 1);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
} }
TEST_F(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) {
// serialize vertex's gid with multiple labels it contains
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
dba.NameToLabel("Ball")};
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
}
// TEST_F(RocksDBStorageTest, GetVerticesByLabel) {
// // search vertices by label
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// memgraph::query::DbAccessor dba(&storage_dba);
// // prepare labels
// std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Player"),
// dba.NameToLabel("Ball")};
// // insert vertices
// for (int i = 0; i < 5; ++i) {
// auto impl = dba.InsertVertex();
// impl.AddLabel(label_ids[i % 3]);
// db.StoreVertex(impl);
// }
// // load vertices from disk
// auto player_vertices = db.Vertices(dba, dba.NameToLabel("Player"));
// auto ball_vertices = db.Vertices(dba, dba.NameToLabel("Ball"));
// ASSERT_EQ(player_vertices.size(), 4);
// ASSERT_EQ(ball_vertices.size(), 1);
// }
TEST_F(RocksDBStorageTest, GetVerticesByProperty) { TEST_F(RocksDBStorageTest, GetVerticesByProperty) {
memgraph::storage::Storage storage; // search vertices by property value
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba); memgraph::query::DbAccessor dba(&storage_dba);
// prepare ssd properties // prepare ssd properties
@ -205,7 +143,6 @@ TEST_F(RocksDBStorageTest, GetVerticesByProperty) {
// prepare hdd properties // prepare hdd properties
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> hdd_properties_1; std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> hdd_properties_1;
hdd_properties_1.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(125.84)); hdd_properties_1.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(125.84));
std::vector properties{ssd_properties_1, ssd_properties_2, hdd_properties_1, hdd_properties_1}; std::vector properties{ssd_properties_1, ssd_properties_2, hdd_properties_1, hdd_properties_1};
// insert vertices // insert vertices
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
@ -227,8 +164,6 @@ TEST_F(RocksDBStorageTest, SerializeEdge) {
// create two vertices and edge between them // create two vertices and edge between them
// search by one of the vertices, return edge // search by one of the vertices, return edge
// check deserialization for both vertices and edge // check deserialization for both vertices and edge
// TODO: storage can also be put in the class
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED); auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba); memgraph::query::DbAccessor dba(&storage_dba);
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Referee")}; std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Referee")};
@ -313,3 +248,54 @@ TEST_F(RocksDBStorageTest, SerializeEdge) {
ASSERT_EQ(*dest_in_edge.Properties(memgraph::storage::View::OLD), ASSERT_EQ(*dest_in_edge.Properties(memgraph::storage::View::OLD),
*src_out_edge.Properties(memgraph::storage::View::OLD)); *src_out_edge.Properties(memgraph::storage::View::OLD));
} }
TEST_F(RocksDBStorageTest, SerializeVertexGIDProperties) {
// serializes vertex's gid, multiple labels and properties
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// prepare labels
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
dba.NameToLabel("Ball")};
// prepare properties
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
// gids
std::unordered_set<uint64_t> gids;
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
memgraph::query::MultiPropsInitChecked(&impl, properties);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
// labels
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
// check properties
auto props = vertex_acc.Properties(memgraph::storage::View::OLD);
ASSERT_FALSE(props.HasError());
auto prop_name = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name"));
auto prop_memory = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory"));
auto prop_price = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price"));
auto prop_unexisting = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random"));
ASSERT_TRUE(prop_name->IsString());
ASSERT_EQ(prop_name->ValueString(), "disk");
ASSERT_TRUE(prop_memory->IsString());
ASSERT_EQ(prop_memory->ValueString(), "1TB");
ASSERT_TRUE(prop_price->IsDouble());
ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21);
ASSERT_TRUE(prop_unexisting->IsNull());
}
}