tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dummy properties serialization

Browse Source
This commit is contained in:
Andi Skrgat 2023-04-05 09:33:47 +02:00
parent e193d4036e
commit 72371cc8ee
3 changed files with 206 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/storage/rocks/storage.hpp
View File

@ -19,12 +19,14 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include "query/common.hpp"
#include "query/db_accessor.hpp"
#include "spdlog/spdlog.h"
#include "storage/rocks/loopback.hpp"
#include "storage/rocks/serialization.hpp"
#include "storage/v2/delta.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/vertex.hpp"
#include "storage/v2/vertex_accessor.hpp"
@ -42,7 +44,7 @@ class RocksDBStorage {
options_.create_if_missing = true;
options_.OptimizeLevelStyleCompaction();
std::filesystem::path rocksdb_path = "./rocks_experiment_test";
if (!memgraph::utils::EnsureDir(rocksdb_path)) {
if (!utils::EnsureDir(rocksdb_path)) {
SPDLOG_ERROR("Unable to create storage folder on disk.");
// TODO: throw some error
}
@ -80,7 +82,7 @@ class RocksDBStorage {
/*
Read all vertices stored in the database.
*/
std::vector<std::unique_ptr<query::VertexAccessor>> Vertices(memgraph::query::DbAccessor &dba) {
std::vector<std::unique_ptr<query::VertexAccessor>> Vertices(query::DbAccessor &dba) {
std::vector<std::unique_ptr<query::VertexAccessor>> vertices;
rocksdb::Iterator *it = db_->NewIterator(rocksdb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
@ -94,7 +96,25 @@ class RocksDBStorage {
}
protected:
std::string SerializeLabels(auto &&labels) {
std::string SerializeProperties(const auto &&properties) {
if (properties.HasError() || (*properties).empty()) {
return "";
}
const auto &properties_iter = (*properties).begin();
auto SerializeEntry = [](const auto &property_entry) {
std::stringstream prop_value;
prop_value << property_entry.second; // PropertyValue has an overload for serialization
return std::to_string(property_entry.first.AsUint()) + ":" + prop_value.str();
};
std::string result = SerializeEntry(*properties_iter);
std::string ser_props = std::accumulate(std::next((*properties).begin()), (*properties).end(), result,
[&SerializeEntry](const std::string &join, const auto &prop_entry) {
return join + "," + SerializeEntry(prop_entry);
});
return ser_props;
}
std::string SerializeLabels(const auto &&labels) {
if (labels.HasError() || (*labels).empty()) {
return "";
}
@ -105,26 +125,26 @@ class RocksDBStorage {
return ser_labels;
}
std::string SerializeGid(const memgraph::storage::Gid &gid) { return std::to_string(gid.AsUint()); }
std::string SerializeGid(const storage::Gid &gid) { return std::to_string(gid.AsUint()); }
std::string SerializeVertex(const query::VertexAccessor &vertex_acc) {
// don't put before serialize labels delimiter
// don't put at the end delimiter
// Serialize labels
std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD));
result += "|";
std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|";
// Serialize gid
result += SerializeGid(vertex_acc.Gid());
result += SerializeGid(vertex_acc.Gid()) + "|";
result += SerializeProperties(vertex_acc.Properties(storage::View::OLD));
spdlog::info("Serialized vertex: {}", result);
return result;
}
std::unique_ptr<query::VertexAccessor> DeserializeVertex(const std::string_view key,
memgraph::query::DbAccessor &dba) {
std::unique_ptr<query::VertexAccessor> DeserializeVertex(const std::string_view key, query::DbAccessor &dba) {
// Create vertex
auto impl = std::make_unique<memgraph::query::VertexAccessor>(dba.InsertVertex());
auto impl = std::make_unique<query::VertexAccessor>(dba.InsertVertex());
spdlog::info("Key to deserialize: {}", key);
const auto vertex_parts = utils::Split(key, "|");
const storage::Gid gid = storage::Gid::FromUint(std::stoull(vertex_parts[1]));
impl->SetGid(gid);
// Deserialize labels
if (!vertex_parts[0].empty()) {
const auto labels = utils::Split(vertex_parts[0], ",");
for (const auto &label : labels) {
@ -144,6 +164,20 @@ class RocksDBStorage {
}
}
}
// deserialize gid
const storage::Gid gid = storage::Gid::FromUint(std::stoull(vertex_parts[1]));
impl->SetGid(gid);
// deserialize properties
const auto ser_properties = utils::Split(vertex_parts[2], ",");
std::map<storage::PropertyId, storage::PropertyValue> vec_properties;
for (const auto &prop_entry : ser_properties) {
const auto &split_prop_entry = utils::Split(prop_entry, ":");
// this is a problem since the value will always contain a string
vec_properties.emplace(storage::PropertyId::FromUint(std::stoull(split_prop_entry[0])),
storage::PropertyValue(split_prop_entry[1]));
}
query::MultiPropsInitChecked(impl.get(), vec_properties);
return impl;
}

3
src/storage/v2/vertex_accessor.cpp
View File

@ -319,6 +319,8 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view
properties = vertex_->properties.Properties();
delta = vertex_->delta;
}
return std::move(properties);
/*
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
switch (delta.action) {
case Delta::Action::SET_PROPERTY: {
@ -356,6 +358,7 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View view
if (!exists) return Error::NONEXISTENT_OBJECT;
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
return std::move(properties);
*/
}
Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const std::vector<EdgeTypeId> &edge_types,

233
tests/unit/storage_rocks.cpp
View File

@ -11,107 +11,188 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <exception>
#include <unordered_set>
#include "query/common.hpp"
#include "query/db_accessor.hpp"
#include "storage/rocks/storage.hpp"
#include "storage/v2/delta.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/view.hpp"
class RocksDBStorageTest : public ::testing::TestWithParam<bool> {};
TEST(RocksDBStorageTest, SerializeVertexGID) {
// TODO: expose transaction of the original storage
memgraph::storage::Storage storage;
memgraph::storage::rocks::RocksDBStorage db;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
std::unordered_set<uint64_t> gids;
for (uint64_t i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
try {
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
std::unordered_set<uint64_t> gids;
for (uint64_t i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
}
} catch (std::exception &) {
db.Clear();
}
}
// TODO: clean labels string
TEST(RocksDBStorageTest, SerializeVertexGIDLabels) {
memgraph::storage::Storage storage;
memgraph::storage::rocks::RocksDBStorage db;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::vector<memgraph::storage::Vertex> vertices;
std::vector<std::string> labels{"Player", "Person", "Ball"};
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids;
for (int i = 0; i < 3; i++) {
label_ids.push_back(dba.NameToLabel(labels[i]));
}
try {
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::vector<std::string> labels{"Player", "Person", "Ball"};
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids;
for (int i = 0; i < 3; i++) {
label_ids.push_back(dba.NameToLabel(labels[i]));
}
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(dba.NameToLabel(labels[i % 3]));
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 1);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(dba.NameToLabel(labels[i % 3]));
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 1);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
} catch (std::exception &) {
db.Clear();
}
}
// TODO: clean labels string
TEST(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) {
memgraph::storage::Storage storage;
memgraph::storage::rocks::RocksDBStorage db;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::vector<memgraph::storage::Vertex> vertices;
std::vector<std::string> labels{"Player", "Person", "Ball"};
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids;
for (int i = 0; i < 3; i++) {
label_ids.push_back(dba.NameToLabel(labels[i]));
}
try {
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// save vertices on disk
std::vector<std::string> labels{"Player", "Person", "Ball"};
std::unordered_set<uint64_t> gids;
std::vector<memgraph::storage::LabelId> label_ids;
for (int i = 0; i < 3; i++) {
label_ids.push_back(dba.NameToLabel(labels[i]));
}
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
} catch (std::exception &e) {
db.Clear();
}
}
TEST(RocksDBStorageTest, SerializeVertexGIDProperties) {
memgraph::storage::rocks::RocksDBStorage db;
try {
memgraph::storage::Storage storage;
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
memgraph::query::DbAccessor dba(&storage_dba);
// prepare labels
std::vector<std::string> labels{"Player", "Person", "Ball"};
std::vector<memgraph::storage::LabelId> label_ids;
for (int i = 0; i < 3; i++) {
label_ids.push_back(dba.NameToLabel(labels[i]));
}
// prepare properties
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
// gids
std::unordered_set<uint64_t> gids;
for (int i = 0; i < 5; ++i) {
gids.insert(i);
auto impl = dba.InsertVertex();
impl.SetGid(memgraph::storage::Gid::FromUint(i));
impl.AddLabel(label_ids[i % 3]);
impl.AddLabel(label_ids[(i + 1) % 3]);
memgraph::query::MultiPropsInitChecked(&impl, properties);
db.StoreVertex(impl);
}
// load vertices from disk
auto loaded_vertices = db.Vertices(dba);
db.Clear();
ASSERT_EQ(loaded_vertices.size(), 5);
for (const auto &vertex_acc : loaded_vertices) {
ASSERT_TRUE(gids.contains(vertex_acc->Gid().AsUint()));
// labels
{
auto labels = vertex_acc->Labels(memgraph::storage::View::OLD);
ASSERT_EQ(labels->size(), 2);
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
}));
}
{
// check properties
auto props = vertex_acc->Properties(memgraph::storage::View::OLD);
ASSERT_FALSE(props.HasError());
auto prop_name = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name"));
auto prop_memory = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory"));
auto prop_price = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price"));
auto prop_unexisting = vertex_acc->GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random"));
ASSERT_TRUE(prop_name->IsString());
ASSERT_EQ(prop_name->ValueString(), "disk");
ASSERT_TRUE(prop_memory->IsString());
ASSERT_EQ(prop_memory->ValueString(), "1TB");
ASSERT_TRUE(prop_price->IsString());
// TODO: needs to be solved
// ASSERT_TRUE(prop_price->IsDouble());
// ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21);
ASSERT_TRUE(prop_unexisting->IsNull());
}
}
} catch (std::exception &e) {
db.Clear();
}
}