tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Serialize cpp types with capnp

Browse Source 
Summary: Serialize shared_ptr, unique_ptr, optional and vector.

Reviewers: teon.banek, msantl

Reviewed By: teon.banek

Subscribers: buda, pullbot

Differential Revision: https://phabricator.memgraph.io/D1364
This commit is contained in:
Marko Culinovic 2018-05-04 10:35:31 +02:00
parent 4b9970ccf8
commit 49d69560e6
6 changed files with 549 additions and 14 deletions

2
src/CMakeLists.txt
View File

@ -114,7 +114,7 @@ endfunction(add_capnp)
add_capnp(query/frontend/semantic/symbol.capnp)
add_capnp(query/frontend/ast/ast.capnp)
add_capnp(utils/typed_value.capnp)
add_capnp(utils/serialization.capnp)
add_capnp(storage/types.capnp)
add_custom_target(generate_capnp DEPENDS ${generated_capnp_files})

2
src/query/frontend/ast/ast.capnp
View File

@ -3,7 +3,7 @@
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("query::capnp");
using Utils = import "/utils/typed_value.capnp";
using Utils = import "/utils/serialization.capnp";
using Storage = import "/storage/types.capnp";
using Symbols = import "/query/frontend/semantic/symbol.capnp";

2
src/query/frontend/ast/ast.cpp
View File

@ -6,7 +6,7 @@
#include "boost/archive/binary_oarchive.hpp"
#include "boost/serialization/export.hpp"
#include "utils/typed_value.capnp.h"
#include "utils/serialization.capnp.h"
namespace query {

84
src/utils/serialization.capnp Normal file
View File

@ -0,0 +1,84 @@
@0xe7647d63b36c2c65;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("utils::capnp");
# Primitive types wrappers.
struct BoxInt32 {
value @0 :Int32;
}
struct BoxInt64 {
value @0 :Int64;
}
struct BoxUInt32 {
value @0 :UInt32;
}
struct BoxUInt64 {
value @0 :UInt32;
}
struct BoxFloat32 {
value @0 :Float32;
}
struct BoxFloat64 {
value @0 :Float64;
}
struct BoxBool {
value @0 :Bool;
}
# CPP Types.
struct Optional(T) {
union {
nullopt @0 :Void;
value @1 :T;
}
}
struct UniquePtr(T) {
union {
nullptr @0 :Void;
value @1 :T;
}
}
struct SharedPtr(T) {
union {
nullptr @0 :Void;
entry @1 :Entry;
}
struct Entry {
id @0 :UInt64;
value @1 :T;
}
}
# TypedValue.
struct TypedValue {
union {
nullType @0 :Void;
bool @1 :Bool;
integer @2 :Int64;
double @3 :Float64;
string @4 :Text;
list @5 :List(TypedValue);
map @6 :List(Entry);
# TODO vertex accessor
# TODO edge accessor
# TODO path
}
struct Entry {
key @0 :Text;
value @1 :TypedValue;
}
}

159
src/utils/serialization.hpp
View File

@ -8,7 +8,7 @@
#include "storage/vertex.hpp"
#include "utils/exceptions.hpp"
#include "typed_value.capnp.h"
#include "utils/serialization.capnp.h"
namespace boost::serialization {
@ -63,7 +63,7 @@ void load(TArchive &ar, std::experimental::optional<T> &opt, unsigned int) {
namespace utils {
inline void SaveCapnpTypedValue(const query::TypedValue &value,
query::capnp::TypedValue::Builder &builder) {
capnp::TypedValue::Builder &builder) {
switch (value.type()) {
case query::TypedValue::Type::Null:
builder.setNullType();
@ -90,31 +90,31 @@ inline void SaveCapnpTypedValue(const query::TypedValue &value,
}
inline void LoadCapnpTypedValue(query::TypedValue &value,
query::capnp::TypedValue::Reader &reader) {
capnp::TypedValue::Reader &reader) {
switch (reader.which()) {
case query::capnp::TypedValue::BOOL:
case capnp::TypedValue::BOOL:
value = reader.getBool();
return;
case query::capnp::TypedValue::DOUBLE:
case capnp::TypedValue::DOUBLE:
value = reader.getDouble();
return;
// case query::capnp::TypedValue::EDGE:
// case capnp::TypedValue::EDGE:
// // TODO
// return;
case query::capnp::TypedValue::INTEGER:
case capnp::TypedValue::INTEGER:
value = reader.getInteger();
return;
case query::capnp::TypedValue::LIST:
case capnp::TypedValue::LIST:
throw utils::NotYetImplemented("Capnp deserialize typed value");
case query::capnp::TypedValue::MAP:
case capnp::TypedValue::MAP:
throw utils::NotYetImplemented("Capnp deserialize typed value");
case query::capnp::TypedValue::NULL_TYPE:
case capnp::TypedValue::NULL_TYPE:
value = query::TypedValue::Null;
return;
// case query::capnp::TypedValue::PATH:
// // TODO
// return;
case query::capnp::TypedValue::STRING:
case capnp::TypedValue::STRING:
value = reader.getString().cStr();
return;
// case query::capnp::TypedValue::VERTEX:
@ -123,6 +123,143 @@ inline void LoadCapnpTypedValue(query::TypedValue &value,
}
}
template <typename T>
inline void SaveVector(const std::vector<T> &data,
typename ::capnp::List<T>::Builder *list_builder) {
for (size_t i = 0; i < data.size(); ++i) {
list_builder->set(i, data[i]);
}
}
template <typename T>
inline void LoadVector(std::vector<T> *data,
const typename ::capnp::List<T>::Reader &list_reader) {
for (const auto e : list_reader) {
data->emplace_back(e);
}
}
template <typename TCapnp, typename T>
inline void SaveVector(
const std::vector<T> &data,
typename ::capnp::List<TCapnp>::Builder *list_builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
for (size_t i = 0; i < data.size(); ++i) {
auto elem_builder = (*list_builder)[i];
save(&elem_builder, data[i]);
}
}
template <typename TCapnp, typename T>
inline void LoadVector(
std::vector<T> *data,
const typename ::capnp::List<TCapnp>::Reader &list_reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load) {
for (const auto reader : list_reader) {
data->emplace_back(load(reader));
}
}
template <typename TCapnp, typename T>
inline void SaveOptional(
const std::experimental::optional<T> &data,
typename capnp::Optional<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
if (data) {
auto value_builder = builder->initValue();
save(&value_builder, data.value());
} else {
builder->setNullopt();
}
}
template <typename TCapnp, typename T>
inline std::experimental::optional<T> LoadOptional(
const typename capnp::Optional<TCapnp>::Reader &reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load) {
switch (reader.which()) {
case capnp::Optional<TCapnp>::NULLOPT:
return std::experimental::nullopt;
case capnp::Optional<TCapnp>::VALUE:
auto value_reader = reader.getValue();
return std::experimental::optional<T>{load(value_reader)};
}
}
template <typename TCapnp, typename T>
inline void SaveUniquePtr(
const std::unique_ptr<T> &data,
typename capnp::UniquePtr<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
if (data) {
auto value_builder = builder->initValue();
save(&value_builder, *data);
} else {
builder->setNullptr();
}
}
template <typename TCapnp, typename T>
inline std::unique_ptr<T> LoadUniquePtr(
const typename capnp::UniquePtr<TCapnp>::Reader &reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load) {
switch (reader.which()) {
case capnp::UniquePtr<TCapnp>::NULLPTR:
return nullptr;
case capnp::UniquePtr<TCapnp>::VALUE:
auto value_reader = reader.getValue();
return std::make_unique<T>(load(value_reader));
}
}
template <typename TCapnp, typename T>
inline void SaveSharedPtr(
const std::shared_ptr<T> &data,
typename capnp::SharedPtr<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save,
std::vector<T *> *saved_pointers) {
if (!data) {
builder->setNullptr();
return;
}
auto entry_builder = builder->initEntry();
auto pointer_id = reinterpret_cast<uintptr_t>(data.get());
CHECK(pointer_id <= std::numeric_limits<uint64_t>::max());
entry_builder.setId(pointer_id);
if (utils::Contains(*saved_pointers, data.get())) {
return;
}
auto value_builder = entry_builder.initValue();
save(&value_builder, *data);
saved_pointers->emplace_back(data.get());
}
template <typename TCapnp, typename T>
inline std::shared_ptr<T> LoadSharedPtr(
const typename capnp::SharedPtr<TCapnp>::Reader &reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load,
std::vector<std::pair<uint64_t, std::shared_ptr<T>>> *loaded_pointers) {
std::shared_ptr<T> ret;
switch (reader.which()) {
case capnp::SharedPtr<TCapnp>::NULLPTR:
ret = nullptr;
break;
case capnp::SharedPtr<TCapnp>::ENTRY:
auto entry_reader = reader.getEntry();
uint64_t pointer_id = entry_reader.getId();
auto found =
std::find_if(loaded_pointers->begin(), loaded_pointers->end(),
[pointer_id](const auto &e) -> bool {
return e.first == pointer_id;
});
if (found != loaded_pointers->end()) return found->second;
auto value_reader = entry_reader.getValue();
ret = std::make_shared<T>(load(value_reader));
loaded_pointers->emplace_back(std::make_pair(pointer_id, ret));
}
return ret;
}
/**
* Saves the given value into the given Boost archive. The optional
* `save_graph_element` function is called if the given `value` is a

314
tests/unit/serialization.cpp
View File

@ -5,6 +5,7 @@
#include "boost/archive/binary_iarchive.hpp"
#include "boost/archive/binary_oarchive.hpp"
#include "capnp/message.h"
#include "utils/serialization.hpp"
using std::experimental::optional;
@ -62,3 +63,316 @@ TEST(Serialization, Tuple) {
EXPECT_EQ(x2, y2);
EXPECT_EQ(x3, y3);
}
void CheckOptionalInt(const std::experimental::optional<int> &x1) {
::capnp::MallocMessageBuilder message;
std::experimental::optional<int> y1;
{
auto builder =
message.initRoot<utils::capnp::Optional<utils::capnp::BoxInt32>>();
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveOptional<utils::capnp::BoxInt32, int>(x1, &builder, save);
}
{
auto reader =
message.getRoot<utils::capnp::Optional<utils::capnp::BoxInt32>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
y1 = utils::LoadOptional<utils::capnp::BoxInt32, int>(reader, load);
}
EXPECT_EQ(x1, y1);
}
TEST(Serialization, CapnpOptional) {
std::experimental::optional<int> x1 = {};
std::experimental::optional<int> x2 = 42;
std::experimental::optional<int> y1, y2;
CheckOptionalInt(x1);
CheckOptionalInt(x2);
}
TEST(Serialization, CapnpOptionalNonCopyable) {
std::experimental::optional<std::unique_ptr<int>> data =
std::make_unique<int>(5);
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::Optional<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](
utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Builder *builder,
const std::unique_ptr<int> &data) {
auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, builder,
save_int);
};
utils::SaveOptional<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save);
}
std::experimental::optional<std::unique_ptr<int>> element;
{
auto reader = message.getRoot<utils::capnp::Optional<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](
const utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Reader &reader)
-> std::unique_ptr<int> {
auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader,
load_int);
};
element =
utils::LoadOptional<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load);
}
EXPECT_EQ(*element.value(), 5);
}
void CheckUniquePtrInt(const std::unique_ptr<int> &x1) {
::capnp::MallocMessageBuilder message;
std::unique_ptr<int> y1;
{
auto builder =
message.initRoot<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>();
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(x1, &builder, save);
}
{
auto reader =
message.getRoot<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
y1 = utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader, load);
}
if (!x1)
EXPECT_EQ(y1, nullptr);
else
EXPECT_EQ(*x1, *y1);
}
TEST(Serialization, CapnpUniquePtr) {
auto x1 = std::make_unique<int>(42);
std::unique_ptr<int> x2;
CheckUniquePtrInt(x1);
CheckUniquePtrInt(x2);
}
TEST(Serialization, CapnpUniquePtrNonCopyable) {
std::unique_ptr<std::unique_ptr<int>> data =
std::make_unique<std::unique_ptr<int>>(std::make_unique<int>(5));
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::UniquePtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](
utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Builder *builder,
const std::unique_ptr<int> &data) {
auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, builder,
save_int);
};
utils::SaveUniquePtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save);
}
std::unique_ptr<std::unique_ptr<int>> element;
{
auto reader = message.getRoot<utils::capnp::UniquePtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](
const utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Reader &reader)
-> std::unique_ptr<int> {
auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader,
load_int);
};
element =
utils::LoadUniquePtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load);
}
EXPECT_EQ(**element, 5);
}
TEST(Serialization, CapnpSharedPtr) {
std::vector<int *> saved_pointers;
auto p1 = std::make_shared<int>(5);
std::shared_ptr<int> p2;
std::vector<std::shared_ptr<int>> pointers{p1, p1, p2};
::capnp::MallocMessageBuilder message;
{
auto builders = message.initRoot<
::capnp::List<utils::capnp::SharedPtr<utils::capnp::BoxInt32>>>(
pointers.size());
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
for (size_t i = 0; i < pointers.size(); ++i) {
auto ptr_builder = builders[i];
utils::SaveSharedPtr<utils::capnp::BoxInt32, int>(
pointers[i], &ptr_builder, save, &saved_pointers);
}
}
EXPECT_EQ(saved_pointers.size(), 1);
std::vector<std::pair<uint64_t, std::shared_ptr<int>>> loaded_pointers;
std::vector<std::shared_ptr<int>> elements;
{
auto reader = message.getRoot<
::capnp::List<utils::capnp::SharedPtr<utils::capnp::BoxInt32>>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
for (const auto ptr_reader : reader) {
elements.emplace_back(utils::LoadSharedPtr<utils::capnp::BoxInt32, int>(
ptr_reader, load, &loaded_pointers));
}
}
EXPECT_EQ(loaded_pointers.size(), 1);
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(*elements[0], 5);
EXPECT_EQ(*elements[0], *elements[1]);
EXPECT_EQ(elements[2].get(), nullptr);
}
TEST(Serialization, CapnpSharedPtrNonCopyable) {
std::shared_ptr<std::unique_ptr<int>> data =
std::make_shared<std::unique_ptr<int>>(std::make_unique<int>(5));
std::vector<std::unique_ptr<int> *> saved_pointers;
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::SharedPtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](
utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Builder *builder,
const std::unique_ptr<int> &data) {
auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, builder,
save_int);
};
utils::SaveSharedPtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save,
&saved_pointers);
}
std::shared_ptr<std::unique_ptr<int>> element;
std::vector<std::pair<uint64_t, std::shared_ptr<std::unique_ptr<int>>>>
loaded_pointers;
{
auto reader = message.getRoot<utils::capnp::SharedPtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](
const utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Reader &reader)
-> std::unique_ptr<int> {
auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader,
load_int);
};
element =
utils::LoadSharedPtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load,
&loaded_pointers);
}
EXPECT_EQ(**element, 5);
}
TEST(Serialization, CapnpVectorPrimitive) {
std::vector<int> data{1, 2, 3};
::capnp::MallocMessageBuilder message;
{
auto list_builder = message.initRoot<::capnp::List<int>>(data.size());
utils::SaveVector<int>(data, &list_builder);
}
std::vector<int> elements;
{
auto reader = message.getRoot<::capnp::List<int>>();
utils::LoadVector<int>(&elements, reader);
}
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(elements[0], 1);
EXPECT_EQ(elements[1], 2);
EXPECT_EQ(elements[2], 3);
}
TEST(Serialization, CapnpVector) {
std::vector<int> data{1, 2, 3};
::capnp::MallocMessageBuilder message;
{
auto list_builder =
message.initRoot<::capnp::List<utils::capnp::BoxInt32>>(data.size());
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveVector<utils::capnp::BoxInt32, int>(data, &list_builder, save);
}
std::vector<int> elements;
{
auto reader = message.getRoot<::capnp::List<utils::capnp::BoxInt32>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
utils::LoadVector<utils::capnp::BoxInt32, int>(&elements, reader, load);
}
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(elements[0], 1);
EXPECT_EQ(elements[1], 2);
EXPECT_EQ(elements[2], 3);
}
TEST(Serialization, CapnpVectorNonCopyable) {
std::vector<std::unique_ptr<int>> data;
data.emplace_back(std::make_unique<int>(5));
data.emplace_back(std::make_unique<int>(10));
::capnp::MallocMessageBuilder message;
{
auto list_builder = message.initRoot<
::capnp::List<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>(
data.size());
auto save = [](
utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Builder *builder,
const std::unique_ptr<int> &data) {
auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, builder,
save_int);
};
utils::SaveVector<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &list_builder, save);
}
std::vector<std::unique_ptr<int>> elements;
{
auto reader = message.getRoot<
::capnp::List<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](
const utils::capnp::UniquePtr<utils::capnp::BoxInt32>::Reader &reader)
-> std::unique_ptr<int> {
auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader,
load_int);
};
utils::LoadVector<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(&elements, reader, load);
}
EXPECT_EQ(elements.size(), 2);
EXPECT_EQ(*elements[0], 5);
EXPECT_EQ(*elements[1], 10);
}