tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix PropertyStore buffer serialization (#1111)

Browse Source
This commit is contained in:
Andi 2023-07-29 19:14:27 +02:00 committed by GitHub
parent 58c0c4cebb
commit 18bd02423a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 101 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/storage/v2/property_store.cpp
View File

@ -1292,12 +1292,11 @@ void PropertyStore::SetBuffer(const std::string_view buffer) {
uint64_t size = 0;
uint8_t *data = nullptr;
size = buffer.size();
if (buffer.size() == sizeof(buffer_) - 1) { // use local buffer
buffer_[0] = kUseLocalBuffer;
size = buffer.size() - 1;
data = &buffer_[1];
} else {
size = buffer.size();
data = new uint8_t[size];
SetSizeData(buffer_, size, data);
}

198
tests/unit/storage_rocks.cpp
View File

@ -9,111 +9,113 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// #include <gmock/gmock.h>
// #include <gtest/gtest.h>
// #include <cassert>
// #include <exception>
// #include <string>
// #include <unordered_set>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <cassert>
#include <exception>
#include <string>
#include <unordered_set>
// #include "query/common.hpp"
// #include "query/db_accessor.hpp"
// #include "storage/v2/delta.hpp"
// #include "storage/v2/disk/storage.hpp"
// #include "storage/v2/id_types.hpp"
// #include "storage/v2/isolation_level.hpp"
// #include "storage/v2/property_value.hpp"
// #include "storage/v2/storage.hpp"
// #include "storage/v2/vertex_accessor.hpp"
// #include "storage/v2/view.hpp"
// #include "utils/rocksdb_serialization.hpp"
#include "disk_test_utils.hpp"
#include "query/common.hpp"
#include "query/db_accessor.hpp"
#include "storage/v2/delta.hpp"
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/property_store.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/view.hpp"
#include "utils/rocksdb_serialization.hpp"
// class RocksDBStorageTest : public ::testing::TestWithParam<bool> {
// public:
// RocksDBStorageTest() { storage = std::unique_ptr<memgraph::storage::Storage>(new memgraph::storage::DiskStorage());
// }
/* Tests that serialization of vertices and edges to RocksDB works correctly.
*/
class RocksDBStorageTest : public ::testing::TestWithParam<bool> {
public:
const std::string testSuite = "storage_rocks";
// ~RocksDBStorageTest() override {}
RocksDBStorageTest() {
config_ = disk_test_utils::GenerateOnDiskConfig(testSuite);
storage = std::make_unique<memgraph::storage::DiskStorage>(config_);
}
// protected:
// std::unique_ptr<memgraph::storage::Storage> storage;
// };
void TearDown() override {
storage.reset(nullptr);
disk_test_utils::RemoveRocksDbDirs(testSuite);
}
// /// Serialize vertex which contains only GID.
// TEST_F(RocksDBStorageTest, SerializeVertexGID) {
// auto acc = storage->Access();
// auto vertex = acc->CreateVertex();
// auto gid = vertex.Gid();
// ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), "|" + memgraph::utils::SerializeIdType(gid));
// }
~RocksDBStorageTest() override {}
/// Serialize vertex with gid and its single label.
// TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
// auto acc = storage->Access();
// auto vertex = acc->CreateVertex();
// auto ser_player_label = acc->NameToLabel("Player");
// auto player_result = vertex.AddLabel(ser_player_label);
// auto gid = vertex.Gid();
// ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_),
// std::to_string(ser_player_label.AsInt()) + "|" + memgraph::utils::SerializeIdType(gid));
// }
protected:
std::unique_ptr<memgraph::storage::Storage> storage;
memgraph::storage::Config config_;
};
// /// Serialize vertex with gid and its multiple labels.
// TEST_F(RocksDBStorageTest, SerializeVertexGIDMultipleLabels) {
// auto acc = storage->Access();
// auto vertex = acc->CreateVertex();
// auto ser_player_label = acc->NameToLabel("Player");
// auto ser_person_label = acc->NameToLabel("Person");
// auto ser_ball_label = acc->NameToLabel("Ball");
// // NOLINTNEXTLINE
// auto player_res = vertex.AddLabel(ser_player_label);
// auto person_res = vertex.AddLabel(ser_person_label);
// auto ball_res = vertex.AddLabel(ser_ball_label);
// auto gid = vertex.Gid();
// ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_),
// std::to_string(ser_player_label.AsInt()) + "," + std::to_string(ser_person_label.AsInt()) + "," +
// std::to_string(ser_ball_label.AsInt()) + "|" + memgraph::utils::SerializeIdType(gid));
// }
TEST_F(RocksDBStorageTest, SerializeVertexGID) {
auto acc = storage->Access();
auto vertex = acc->CreateVertex();
auto gid = vertex.Gid();
ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), "|" + memgraph::utils::SerializeIdType(gid));
}
// /// Serialize edge.
// TEST_F(RocksDBStorageTest, SerializeEdge) {
// auto acc = storage->Access();
// auto vertex1 = acc->CreateVertex();
// auto vertex2 = acc->CreateVertex();
// auto edge = acc->CreateEdge(&vertex1, &vertex2, acc->NameToEdgeType("KNOWS"));
// auto gid = edge->Gid();
// auto ser_result = memgraph::utils::SerializeEdge(vertex1.Gid(), vertex2.Gid(), edge->EdgeType(), edge->edge_.ptr);
// ASSERT_EQ(ser_result.first,
// memgraph::utils::SerializeIdType(vertex1.Gid()) + "|" + memgraph::utils::SerializeIdType(vertex2.Gid()) +
// "|0|" + std::to_string(edge->EdgeType().AsInt()) + "|" + memgraph::utils::SerializeIdType(gid));
// ASSERT_EQ(ser_result.second,
// memgraph::utils::SerializeIdType(vertex2.Gid()) + "|" + memgraph::utils::SerializeIdType(vertex1.Gid()) +
// "|1|" + std::to_string(edge->EdgeType().AsInt()) + "|" + memgraph::utils::SerializeIdType(gid));
// }
TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
auto acc = storage->Access();
auto vertex = acc->CreateVertex();
auto ser_player_label = acc->NameToLabel("Player");
auto ser_user_label = acc->NameToLabel("User");
ASSERT_FALSE(vertex.AddLabel(ser_player_label).HasError());
ASSERT_FALSE(vertex.AddLabel(ser_user_label).HasError());
auto gid = vertex.Gid();
ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), std::to_string(ser_player_label.AsInt()) + "," +
std::to_string(ser_user_label.AsInt()) + "|" +
memgraph::utils::SerializeIdType(gid));
}
// TEST_F(RocksDBStorageTest, DeserializeVertex) {
// NOTE: This test would fail in the case of snaphsot isolation because of the way in which RocksDB
// serializes commit timestamp.
// const char *serialized_vertex = "1|1";
// auto acc = storage->Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// auto *acc_ptr = static_cast<memgraph::storage::DiskStorage::DiskAccessor *>(acc.get());
// const char *value = "garbage";
// auto vertex = acc_ptr->DeserializeVertex(serialized_vertex, value);
// ASSERT_EQ(vertex->Gid().AsInt(), 1);
// ASSERT_EQ(*vertex->HasLabel(memgraph::storage::LabelId::FromUint(1), memgraph::storage::View::OLD), true);
// }
TEST_F(RocksDBStorageTest, SerializePropertiesLocalBuffer) {
memgraph::storage::PropertyStore props;
auto id = memgraph::storage::PropertyId::FromInt(0);
auto id_value = memgraph::storage::PropertyValue(1);
auto completion_percentage = memgraph::storage::PropertyId::FromInt(1);
auto completion_percentage_value = memgraph::storage::PropertyValue(14);
auto gender = memgraph::storage::PropertyId::FromInt(2);
auto gender_value = memgraph::storage::PropertyValue("man");
auto age = memgraph::storage::PropertyId::FromInt(3);
auto age_value = memgraph::storage::PropertyValue(26);
ASSERT_TRUE(props.SetProperty(id, id_value));
ASSERT_TRUE(props.SetProperty(age, age_value));
ASSERT_TRUE(props.SetProperty(completion_percentage, completion_percentage_value));
ASSERT_TRUE(props.SetProperty(gender, gender_value));
std::string serialized_props = memgraph::utils::SerializeProperties(props);
memgraph::storage::PropertyStore deserialized_props =
memgraph::storage::PropertyStore::CreateFromBuffer(serialized_props);
// TEST_F(RocksDBStorageTest, DeserializeEdge) {
// NOTE: This test would fail in the case of snaphsot isolation because of the way in which RocksDB
// serializes commit timestamp.
// auto acc = storage->Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
// auto vertex1 = acc->CreateVertex();
// auto vertex2 = acc->CreateVertex();
// auto serialized_edge = fmt::format("{}|{}|0|1|2", vertex1.Gid().AsInt(), vertex2.Gid().AsInt());
// auto acc_ptr = static_cast<memgraph::storage::DiskStorage::DiskAccessor *>(acc.get());
// auto edge = acc_ptr->DeserializeEdge(serialized_edge, "garbage");
// ASSERT_EQ(edge->Gid().AsInt(), 2);
// ASSERT_EQ(edge->EdgeType().AsInt(), 1);
// ASSERT_EQ(edge->from_vertex_->gid.AsInt(), vertex1.Gid().AsInt());
// ASSERT_EQ(edge->to_vertex_->gid.AsInt(), vertex2.Gid().AsInt());
// }
for (const auto &[prop_id, prop_value] : props.Properties()) {
ASSERT_TRUE(deserialized_props.IsPropertyEqual(prop_id, prop_value));
}
}
TEST_F(RocksDBStorageTest, SerializePropertiesExternalBuffer) {
memgraph::storage::PropertyStore props;
auto id = memgraph::storage::PropertyId::FromInt(0);
auto id_value = memgraph::storage::PropertyValue(1);
auto completion_percentage = memgraph::storage::PropertyId::FromInt(1);
auto completion_percentage_value = memgraph::storage::PropertyValue(14);
auto gender = memgraph::storage::PropertyId::FromInt(2);
// Use big value so that memory for properties is allocated on the heap not on the stack
auto gender_value = memgraph::storage::PropertyValue("man167863816386826");
auto age = memgraph::storage::PropertyId::FromInt(3);
auto age_value = memgraph::storage::PropertyValue(26);
ASSERT_TRUE(props.SetProperty(id, id_value));
ASSERT_TRUE(props.SetProperty(age, age_value));
ASSERT_TRUE(props.SetProperty(completion_percentage, completion_percentage_value));
ASSERT_TRUE(props.SetProperty(gender, gender_value));
std::string serialized_props = memgraph::utils::SerializeProperties(props);
memgraph::storage::PropertyStore deserialized_props =
memgraph::storage::PropertyStore::CreateFromBuffer(serialized_props);
for (const auto &[prop_id, prop_value] : props.Properties()) {
ASSERT_TRUE(deserialized_props.IsPropertyEqual(prop_id, prop_value));
}
}