Initial version of custom serialization

Reviewers: teon.banek, buda

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1711

src/communication/rpc/serialization.hpp

@@ -0,0 +1,362 @@
+#pragma once
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <experimental/optional>
+#include <experimental/type_traits>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "communication/rpc/streams.hpp"
+
+// The namespace name stands for SaveLoadKit. It should be not mistaken for the
+// Mercedes car model line.
+namespace slk {
+
+// Static assert for the assumption made in this library.
+static_assert(std::experimental::is_same_v<std::uint8_t, char> ||
+                  std::experimental::is_same_v<std::uint8_t, unsigned char>,
+              "The slk library requires uint8_t to be implemented as char or "
+              "unsigned char.");
+
+// Forward declarations for all recursive `Save` and `Load` functions must be
+// here because C++ doesn't know how to resolve the function call if it isn't in
+// the global namespace.
+
+template <typename T>
+void Save(const std::vector<T> &obj, Builder *builder);
+template <typename T>
+void Load(std::vector<T> *obj, Reader *reader);
+
+template <typename T>
+void Save(const std::set<T> &obj, Builder *builder);
+template <typename T>
+void Load(std::set<T> *obj, Reader *reader);
+
+template <typename K, typename V>
+void Save(const std::map<K, V> &obj, Builder *builder);
+template <typename K, typename V>
+void Load(std::map<K, V> *obj, Reader *reader);
+
+template <typename K, typename V>
+void Save(const std::unordered_map<K, V> &obj, Builder *builder);
+template <typename K, typename V>
+void Load(std::unordered_map<K, V> *obj, Reader *reader);
+
+template <typename T>
+void Save(const std::unique_ptr<T> &obj, Builder *builder);
+template <typename T>
+void Load(std::unique_ptr<T> *obj, Reader *reader);
+
+template <typename T>
+void Save(const std::experimental::optional<T> &obj, Builder *builder);
+template <typename T>
+void Load(std::experimental::optional<T> *obj, Reader *reader);
+
+template <typename T>
+void Save(const std::shared_ptr<T> &obj, Builder *builder,
+          std::vector<T *> *saved);
+template <typename T>
+void Load(std::shared_ptr<T> *obj, Reader *reader,
+          std::vector<std::shared_ptr<T>> *loaded);
+
+// Implementation of serialization for primitive types.
+
+#define MAKE_PRIMITIVE_SAVE(primitive_type)                \
+  inline void Save(primitive_type obj, Builder *builder) { \
+    builder->Save(reinterpret_cast<const uint8_t *>(&obj), \
+                  sizeof(primitive_type));                 \
+  }
+
+MAKE_PRIMITIVE_SAVE(bool)
+MAKE_PRIMITIVE_SAVE(int8_t)
+MAKE_PRIMITIVE_SAVE(uint8_t)
+MAKE_PRIMITIVE_SAVE(int16_t)
+MAKE_PRIMITIVE_SAVE(uint16_t)
+MAKE_PRIMITIVE_SAVE(int32_t)
+MAKE_PRIMITIVE_SAVE(uint32_t)
+MAKE_PRIMITIVE_SAVE(int64_t)
+MAKE_PRIMITIVE_SAVE(uint64_t)
+MAKE_PRIMITIVE_SAVE(float)
+MAKE_PRIMITIVE_SAVE(double)
+
+#undef MAKE_PRIMITIVE_SAVE
+
+#define MAKE_PRIMITIVE_LOAD(primitive_type)                                 \
+  inline void Load(primitive_type *obj, Reader *reader) {                   \
+    reader->Load(reinterpret_cast<uint8_t *>(obj), sizeof(primitive_type)); \
+  }
+
+MAKE_PRIMITIVE_LOAD(bool)
+MAKE_PRIMITIVE_LOAD(int8_t)
+MAKE_PRIMITIVE_LOAD(uint8_t)
+MAKE_PRIMITIVE_LOAD(int16_t)
+MAKE_PRIMITIVE_LOAD(uint16_t)
+MAKE_PRIMITIVE_LOAD(int32_t)
+MAKE_PRIMITIVE_LOAD(uint32_t)
+MAKE_PRIMITIVE_LOAD(int64_t)
+MAKE_PRIMITIVE_LOAD(uint64_t)
+MAKE_PRIMITIVE_LOAD(float)
+MAKE_PRIMITIVE_LOAD(double)
+
+#undef MAKE_PRIMITIVE_LOAD
+
+// Implementation of serialization of complex types.
+
+inline void Save(const std::string &obj, Builder *builder) {
+  uint64_t size = obj.size();
+  builder->Save(reinterpret_cast<const uint8_t *>(&size), sizeof(uint64_t));
+  builder->Save(reinterpret_cast<const uint8_t *>(obj.data()), size);
+}
+
+inline void Load(std::string *obj, Reader *reader) {
+  const int kMaxStackBuffer = 8192;
+  uint64_t size = 0;
+  reader->Load(reinterpret_cast<uint8_t *>(&size), sizeof(uint64_t));
+  if (size < kMaxStackBuffer) {
+    // Here we use a temporary buffer on the stack to prevent temporary
+    // allocations. Most of strings that are decoded are small so it makes no
+    // sense to allocate a temporary buffer every time we decode a string. This
+    // way we allocate a temporary buffer only when the string is large. This
+    // wouldn't be necessary if we had full C++17 support. In C++17 we could
+    // preallocate the `buff[size]` in the destination string `*obj =
+    // std::string('\0', size)` and just call `reader->Load(obj->data())`.
+    char buff[kMaxStackBuffer];
+    reader->Load(reinterpret_cast<uint8_t *>(buff), size);
+    *obj = std::string(buff, size);
+  } else {
+    auto buff = std::unique_ptr<char[]>(new char[size]);
+    reader->Load(reinterpret_cast<uint8_t *>(buff.get()), size);
+    *obj = std::string(buff.get(), size);
+  }
+}
+
+template <typename T>
+inline void Save(const std::vector<T> &obj, Builder *builder) {
+  uint64_t size = obj.size();
+  Save(size, builder);
+  for (const auto &item : obj) {
+    Save(item, builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::vector<T> *obj, Reader *reader) {
+  uint64_t size = 0;
+  Load(&size, reader);
+  obj->resize(size);
+  for (uint64_t i = 0; i < size; ++i) {
+    Load(&(*obj)[i], reader);
+  }
+}
+
+template <typename T>
+inline void Save(const std::set<T> &obj, Builder *builder) {
+  uint64_t size = obj.size();
+  Save(size, builder);
+  for (const auto &item : obj) {
+    Save(item, builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::set<T> *obj, Reader *reader) {
+  uint64_t size = 0;
+  Load(&size, reader);
+  for (uint64_t i = 0; i < size; ++i) {
+    T item;
+    Load(&item, reader);
+    obj->emplace(std::move(item));
+  }
+}
+
+#define MAKE_MAP_SAVE(map_type)                                   \
+  template <typename K, typename V>                               \
+  inline void Save(const map_type<K, V> &obj, Builder *builder) { \
+    uint64_t size = obj.size();                                   \
+    Save(size, builder);                                          \
+    for (const auto &item : obj) {                                \
+      Save(item.first, builder);                                  \
+      Save(item.second, builder);                                 \
+    }                                                             \
+  }
+
+MAKE_MAP_SAVE(std::map)
+MAKE_MAP_SAVE(std::unordered_map)
+
+#undef MAKE_MAP_SAVE
+
+#define MAKE_MAP_LOAD(map_type)                           \
+  template <typename K, typename V>                       \
+  inline void Load(map_type<K, V> *obj, Reader *reader) { \
+    uint64_t size = 0;                                    \
+    Load(&size, reader);                                  \
+    for (uint64_t i = 0; i < size; ++i) {                 \
+      K key;                                              \
+      V value;                                            \
+      Load(&key, reader);                                 \
+      Load(&value, reader);                               \
+      obj->emplace(std::move(key), std::move(value));     \
+    }                                                     \
+  }
+
+MAKE_MAP_LOAD(std::map)
+MAKE_MAP_LOAD(std::unordered_map)
+
+#undef MAKE_MAP_LOAD
+
+template <typename T>
+inline void Save(const std::unique_ptr<T> &obj, Builder *builder) {
+  if (obj.get() == nullptr) {
+    bool exists = false;
+    Save(exists, builder);
+  } else {
+    bool exists = true;
+    Save(exists, builder);
+    Save(*obj.get(), builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::unique_ptr<T> *obj, Reader *reader) {
+  bool exists = false;
+  Load(&exists, reader);
+  if (exists) {
+    T item;
+    Load(&item, reader);
+    *obj = std::make_unique<T>(std::move(item));
+  } else {
+    *obj = nullptr;
+  }
+}
+
+template <typename T>
+inline void Save(const std::experimental::optional<T> &obj, Builder *builder) {
+  if (obj == std::experimental::nullopt) {
+    bool exists = false;
+    Save(exists, builder);
+  } else {
+    bool exists = true;
+    Save(exists, builder);
+    Save(*obj, builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::experimental::optional<T> *obj, Reader *reader) {
+  bool exists = false;
+  Load(&exists, reader);
+  if (exists) {
+    T item;
+    Load(&item, reader);
+    obj->emplace(std::move(item));
+  } else {
+    *obj = std::experimental::nullopt;
+  }
+}
+
+// Implementation of three argument serialization for complex types.
+
+template <typename T>
+inline void Save(const std::shared_ptr<T> &obj, Builder *builder,
+                 std::vector<T *> *saved) {
+  if (obj.get() == nullptr) {
+    bool exists = false;
+    Save(exists, builder);
+  } else {
+    bool exists = true;
+    Save(exists, builder);
+    auto pos = std::find(saved->begin(), saved->end(), obj.get());
+    if (pos != saved->end()) {
+      bool in_place = false;
+      Save(in_place, builder);
+      uint64_t index = pos - saved->begin();
+      Save(index, builder);
+    } else {
+      bool in_place = true;
+      Save(in_place, builder);
+      Save(*obj.get(), builder);
+      saved->push_back(obj.get());
+    }
+  }
+}
+
+template <typename T>
+inline void Load(std::shared_ptr<T> *obj, Reader *reader,
+                 std::vector<std::shared_ptr<T>> *loaded) {
+  bool exists = false;
+  Load(&exists, reader);
+  if (exists) {
+    bool in_place = false;
+    Load(&in_place, reader);
+    if (in_place) {
+      T item;
+      Load(&item, reader);
+      *obj = std::make_shared<T>(std::move(item));
+      loaded->push_back(*obj);
+    } else {
+      uint64_t index = 0;
+      Load(&index, reader);
+      // TODO: handle if index doesn't exist!
+      *obj = (*loaded)[index];
+    }
+  } else {
+    *obj = nullptr;
+  }
+}
+
+template <typename T>
+inline void Save(const std::vector<T> &obj, Builder *builder,
+                 std::function<void(const T &, Builder *)> item_save_function) {
+  uint64_t size = obj.size();
+  Save(size, builder);
+  for (const auto &item : obj) {
+    item_save_function(item, builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::vector<T> *obj, Reader *reader,
+                 std::function<void(T *, Reader *)> item_load_function) {
+  uint64_t size = 0;
+  Load(&size, reader);
+  obj->resize(size);
+  for (uint64_t i = 0; i < size; ++i) {
+    item_load_function(&(*obj)[i], reader);
+  }
+}
+
+template <typename T>
+inline void Save(const std::experimental::optional<T> &obj, Builder *builder,
+                 std::function<void(const T &, Builder *)> item_save_function) {
+  if (obj == std::experimental::nullopt) {
+    bool exists = false;
+    Save(exists, builder);
+  } else {
+    bool exists = true;
+    Save(exists, builder);
+    item_save_function(*obj, builder);
+  }
+}
+
+template <typename T>
+inline void Load(std::experimental::optional<T> *obj, Reader *reader,
+                 std::function<void(T *, Reader *)> item_load_function) {
+  bool exists = false;
+  Load(&exists, reader);
+  if (exists) {
+    T item;
+    item_load_function(&item, reader);
+    obj->emplace(std::move(item));
+  } else {
+    *obj = std::experimental::nullopt;
+  }
+}
+}  // namespace slk

src/communication/rpc/streams.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <cstdint>
+
+#include <glog/logging.h>
+
+namespace slk {
+
+// TODO (mferencevic): Implementations of the `Builder` and `Reader` are just
+// mock implementations for now. They will be finished when they will be
+// integrated into the RPC layer.
+
+class Builder {
+ public:
+  void Save(const uint8_t *data, uint64_t size) {
+    CHECK(size_ + size <= 262144);
+    memcpy(data_ + size_, data, size);
+    size_ += size;
+  }
+
+  uint8_t *data() { return data_; }
+  uint64_t size() { return size_; }
+
+ private:
+  uint8_t data_[262144];
+  uint64_t size_{0};
+};
+
+class Reader {
+ public:
+  Reader(const uint8_t *data, uint64_t size) : data_(data), size_(size) {}
+
+  void Load(uint8_t *data, uint64_t size) {
+    CHECK(offset_ <= size_);
+    memcpy(data, data_ + offset_, size);
+    offset_ += size;
+  }
+
+ private:
+  const uint8_t *data_;
+  uint64_t size_;
+  uint64_t offset_{0};
+};
+
+}  // namespace slk

tests/benchmark/serialization.cpp

@@ -9,6 +9,8 @@
 #include "query/frontend/semantic/symbol.capnp.h"
 #include "query/frontend/semantic/symbol.hpp"
 
+#include "communication/rpc/serialization.hpp"
+
 class SymbolVectorFixture : public benchmark::Fixture {
  protected:
   std::vector<query::Symbol> symbols_;
@@ -93,4 +95,112 @@ BENCHMARK_REGISTER_F(SymbolVectorFixture, CapnpDeserial)
     ->Range(4, 1 << 12)
     ->Unit(benchmark::kNanosecond);
 
+uint8_t Type2Int(query::Symbol::Type type) {
+  switch (type) {
+    case query::Symbol::Type::Any:
+      return 1;
+    case query::Symbol::Type::Vertex:
+      return 2;
+    case query::Symbol::Type::Edge:
+      return 3;
+    case query::Symbol::Type::Path:
+      return 4;
+    case query::Symbol::Type::Number:
+      return 5;
+    case query::Symbol::Type::EdgeList:
+      return 6;
+  }
+}
+
+query::Symbol::Type Int2Type(uint8_t value) {
+  switch (value) {
+    case 1:
+      return query::Symbol::Type::Any;
+    case 2:
+      return query::Symbol::Type::Vertex;
+    case 3:
+      return query::Symbol::Type::Edge;
+    case 4:
+      return query::Symbol::Type::Path;
+    case 5:
+      return query::Symbol::Type::Number;
+    case 6:
+      return query::Symbol::Type::EdgeList;
+  }
+  CHECK(false);
+}
+
+namespace slk {
+void Save(const query::Symbol &obj, slk::Builder *builder) {
+  Save(obj.name(), builder);
+  Save(obj.position(), builder);
+  Save(Type2Int(obj.type()), builder);
+  Save(obj.user_declared(), builder);
+  Save(obj.token_position(), builder);
+}
+
+void Load(query::Symbol *obj, slk::Reader *reader) {
+  std::string name;
+  Load(&name, reader);
+  int position = 0;
+  Load(&position, reader);
+  uint8_t type = 0;
+  Load(&type, reader);
+  bool user_declared = false;
+  Load(&user_declared, reader);
+  int token_position = 0;
+  Load(&token_position, reader);
+  *obj = query::Symbol(std::move(name), position, user_declared, Int2Type(type),
+                       token_position);
+}
+}  // namespace slk
+
+void SymbolVectorToCustom(const std::vector<query::Symbol> &symbols,
+                          slk::Builder *builder) {
+  slk::Save(symbols.size(), builder);
+  for (int i = 0; i < symbols.size(); ++i) {
+    slk::Save(symbols[i], builder);
+  }
+}
+
+void CustomToSymbolVector(std::vector<query::Symbol> *symbols,
+                          slk::Reader *reader) {
+  uint64_t size = 0;
+  slk::Load(&size, reader);
+  symbols->resize(size);
+  for (uint64_t i = 0; i < size; ++i) {
+    slk::Load(&(*symbols)[i], reader);
+  }
+}
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, CustomSerial)(benchmark::State &state) {
+  while (state.KeepRunning()) {
+    slk::Builder builder;
+    SymbolVectorToCustom(symbols_, &builder);
+  }
+}
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, CustomDeserial)
+(benchmark::State &state) {
+  while (state.KeepRunning()) {
+    state.PauseTiming();
+    slk::Builder builder;
+    SymbolVectorToCustom(symbols_, &builder);
+    state.ResumeTiming();
+    slk::Reader reader(builder.data(), builder.size());
+    std::vector<query::Symbol> symbols;
+    CustomToSymbolVector(&symbols, &reader);
+  }
+}
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, CustomSerial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, CustomDeserial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
 BENCHMARK_MAIN();

tests/unit/CMakeLists.txt

@@ -11,7 +11,7 @@ function(add_unit_test test_cpp)
   # used to help create two targets of the same name even though CMake
   # requires unique logical target names
   set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
-  target_link_libraries(${target_name} gtest gmock gtest_main)
+  target_link_libraries(${target_name} gtest gmock gtest_main Threads::Threads)
   # register test
   set(output_path ${CMAKE_BINARY_DIR}/test_results/unit/${target_name}.xml)
   add_test(${target_name} ${exec_name} --gtest_output=xml:${output_path})
@@ -37,6 +37,10 @@ target_link_libraries(${test_prefix}concurrent_map mg-single-node kvstore_dummy_
 add_unit_test(counters.cpp)
 target_link_libraries(${test_prefix}counters mg-distributed kvstore_dummy_lib)
 
+# TODO (mferencevic): remove glog and gflags
+add_unit_test(custom_serialization.cpp)
+target_link_libraries(${test_prefix}custom_serialization glog gflags)
+
 add_unit_test(cypher_main_visitor.cpp)
 target_link_libraries(${test_prefix}cypher_main_visitor mg-single-node kvstore_dummy_lib)
 

tests/unit/custom_serialization.cpp

@@ -0,0 +1,460 @@
+#include <gtest/gtest.h>
+
+#include "communication/rpc/serialization.hpp"
+
+#define CREATE_PRIMITIVE_TEST(primitive_type, original_value, decoded_value) \
+  {                                                                          \
+    ASSERT_NE(original_value, decoded_value);                                \
+    primitive_type original = original_value;                                \
+    slk::Builder builder;                                                    \
+    slk::Save(original, &builder);                                           \
+    ASSERT_EQ(builder.size(), sizeof(primitive_type));                       \
+    primitive_type decoded = decoded_value;                                  \
+    slk::Reader reader(builder.data(), builder.size());                      \
+    slk::Load(&decoded, &reader);                                            \
+    ASSERT_EQ(original, decoded);                                            \
+  }
+
+TEST(CustomSerialization, Primitive) {
+  CREATE_PRIMITIVE_TEST(bool, true, false);
+  CREATE_PRIMITIVE_TEST(int8_t, 0x12, 0);
+  CREATE_PRIMITIVE_TEST(uint8_t, 0x12, 0);
+  CREATE_PRIMITIVE_TEST(int16_t, 0x1234, 0);
+  CREATE_PRIMITIVE_TEST(uint16_t, 0x1234, 0);
+  CREATE_PRIMITIVE_TEST(int32_t, 0x12345678, 0);
+  CREATE_PRIMITIVE_TEST(uint32_t, 0x12345678, 0);
+  CREATE_PRIMITIVE_TEST(int64_t, 0x1234567890abcdef, 0);
+  CREATE_PRIMITIVE_TEST(uint64_t, 0x1234567890abcdef, 0);
+  CREATE_PRIMITIVE_TEST(float, 1.23456789, 0);
+  CREATE_PRIMITIVE_TEST(double, 1234567890.1234567890, 0);
+}
+
+TEST(CustomSerialization, String) {
+  std::string original = "hello world";
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(uint64_t) + original.size());
+  std::string decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, VectorPrimitive) {
+  std::vector<int> original{1, 2, 3, 4, 5};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(uint64_t) + original.size() * sizeof(int));
+  std::vector<int> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, VectorString) {
+  std::vector<std::string> original{"hai hai hai", "nandare!"};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  uint64_t size = sizeof(uint64_t);
+  for (const auto &item : original) {
+    size += sizeof(uint64_t) + item.size();
+  }
+  ASSERT_EQ(builder.size(), size);
+  std::vector<std::string> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, SetPrimitive) {
+  std::set<int> original{1, 2, 3, 4, 5};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(uint64_t) + original.size() * sizeof(int));
+  std::set<int> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, SetString) {
+  std::set<std::string> original{"hai hai hai", "nandare!"};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  uint64_t size = sizeof(uint64_t);
+  for (const auto &item : original) {
+    size += sizeof(uint64_t) + item.size();
+  }
+  ASSERT_EQ(builder.size(), size);
+  std::set<std::string> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, MapPrimitive) {
+  std::map<int, int> original{{1, 2}, {3, 4}, {5, 6}};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(),
+            sizeof(uint64_t) + original.size() * sizeof(int) * 2);
+  std::map<int, int> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, MapString) {
+  std::map<std::string, std::string> original{{"hai hai hai", "nandare!"}};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  uint64_t size = sizeof(uint64_t);
+  for (const auto &item : original) {
+    size += sizeof(uint64_t) + item.first.size();
+    size += sizeof(uint64_t) + item.second.size();
+  }
+  ASSERT_EQ(builder.size(), size);
+  std::map<std::string, std::string> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, UnorderedMapPrimitive) {
+  std::unordered_map<int, int> original{{1, 2}, {3, 4}, {5, 6}};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(),
+            sizeof(uint64_t) + original.size() * sizeof(int) * 2);
+  std::unordered_map<int, int> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, UnorderedMapString) {
+  std::unordered_map<std::string, std::string> original{
+      {"hai hai hai", "nandare!"}};
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  uint64_t size = sizeof(uint64_t);
+  for (const auto &item : original) {
+    size += sizeof(uint64_t) + item.first.size();
+    size += sizeof(uint64_t) + item.second.size();
+  }
+  ASSERT_EQ(builder.size(), size);
+  std::unordered_map<std::string, std::string> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, UniquePtrEmpty) {
+  std::unique_ptr<std::string> original;
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(bool));
+  std::unique_ptr<std::string> decoded =
+      std::make_unique<std::string>("nandare!");
+  ASSERT_NE(decoded.get(), nullptr);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(decoded.get(), nullptr);
+}
+
+TEST(CustomSerialization, UniquePtrFull) {
+  std::unique_ptr<std::string> original =
+      std::make_unique<std::string>("nandare!");
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(),
+            sizeof(bool) + sizeof(uint64_t) + original.get()->size());
+  std::unique_ptr<std::string> decoded;
+  ASSERT_EQ(decoded.get(), nullptr);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_NE(decoded.get(), nullptr);
+  ASSERT_EQ(*original.get(), *decoded.get());
+}
+
+TEST(CustomSerialization, OptionalPrimitiveEmpty) {
+  std::experimental::optional<int> original;
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(bool));
+  std::experimental::optional<int> decoded = 5;
+  ASSERT_NE(decoded, std::experimental::nullopt);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(decoded, std::experimental::nullopt);
+}
+
+TEST(CustomSerialization, OptionalPrimitiveFull) {
+  std::experimental::optional<int> original = 5;
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(bool) + sizeof(int));
+  std::experimental::optional<int> decoded;
+  ASSERT_EQ(decoded, std::experimental::nullopt);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_NE(decoded, std::experimental::nullopt);
+  ASSERT_EQ(*original, *decoded);
+}
+
+TEST(CustomSerialization, OptionalStringEmpty) {
+  std::experimental::optional<std::string> original;
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(bool));
+  std::experimental::optional<std::string> decoded = "nandare!";
+  ASSERT_NE(decoded, std::experimental::nullopt);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(decoded, std::experimental::nullopt);
+}
+
+TEST(CustomSerialization, OptionalStringFull) {
+  std::experimental::optional<std::string> original = "nandare!";
+  slk::Builder builder;
+  slk::Save(original, &builder);
+  ASSERT_EQ(builder.size(), sizeof(bool) + sizeof(uint64_t) + original->size());
+  std::experimental::optional<std::string> decoded;
+  ASSERT_EQ(decoded, std::experimental::nullopt);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_NE(decoded, std::experimental::nullopt);
+  ASSERT_EQ(*original, *decoded);
+}
+
+TEST(CustomSerialization, SharedPtrEmpty) {
+  std::shared_ptr<std::string> original;
+  std::vector<std::string *> saved;
+  slk::Builder builder;
+  slk::Save(original, &builder, &saved);
+  ASSERT_EQ(builder.size(), sizeof(bool));
+  std::shared_ptr<std::string> decoded =
+      std::make_shared<std::string>("nandare!");
+  std::vector<std::shared_ptr<std::string>> loaded;
+  ASSERT_NE(decoded.get(), nullptr);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader, &loaded);
+  ASSERT_EQ(decoded.get(), nullptr);
+  ASSERT_EQ(saved.size(), 0);
+  ASSERT_EQ(loaded.size(), 0);
+}
+
+TEST(CustomSerialization, SharedPtrFull) {
+  std::shared_ptr<std::string> original =
+      std::make_shared<std::string>("nandare!");
+  std::vector<std::string *> saved;
+  slk::Builder builder;
+  slk::Save(original, &builder, &saved);
+  ASSERT_EQ(builder.size(),
+            sizeof(bool) * 2 + sizeof(uint64_t) + original.get()->size());
+  std::shared_ptr<std::string> decoded;
+  std::vector<std::shared_ptr<std::string>> loaded;
+  ASSERT_EQ(decoded.get(), nullptr);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader, &loaded);
+  ASSERT_NE(decoded.get(), nullptr);
+  ASSERT_EQ(*original.get(), *decoded.get());
+  ASSERT_EQ(saved.size(), 1);
+  ASSERT_EQ(loaded.size(), 1);
+}
+
+TEST(CustomSerialization, SharedPtrMultiple) {
+  std::shared_ptr<std::string> ptr1 = std::make_shared<std::string>("nandare!");
+  std::shared_ptr<std::string> ptr2;
+  std::shared_ptr<std::string> ptr3 =
+      std::make_shared<std::string>("hai hai hai");
+  std::vector<std::string *> saved;
+
+  slk::Builder builder;
+  slk::Save(ptr1, &builder, &saved);
+  slk::Save(ptr2, &builder, &saved);
+  slk::Save(ptr3, &builder, &saved);
+  slk::Save(ptr1, &builder, &saved);
+  slk::Save(ptr3, &builder, &saved);
+
+  // clang-format off
+  ASSERT_EQ(builder.size(),
+            sizeof(bool) * 2 + sizeof(uint64_t) + ptr1.get()->size() +
+            sizeof(bool) +
+            sizeof(bool) * 2 + sizeof(uint64_t) + ptr3.get()->size() +
+            sizeof(bool) * 2 + sizeof(uint64_t) +
+            sizeof(bool) * 2 + sizeof(uint64_t));
+  // clang-format on
+
+  std::shared_ptr<std::string> dec1, dec2, dec3, dec4, dec5;
+  std::vector<std::shared_ptr<std::string>> loaded;
+
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&dec1, &reader, &loaded);
+  slk::Load(&dec2, &reader, &loaded);
+  slk::Load(&dec3, &reader, &loaded);
+  slk::Load(&dec4, &reader, &loaded);
+  slk::Load(&dec5, &reader, &loaded);
+
+  ASSERT_EQ(saved.size(), 2);
+  ASSERT_EQ(loaded.size(), 2);
+
+  ASSERT_NE(dec1.get(), nullptr);
+  ASSERT_EQ(dec2.get(), nullptr);
+  ASSERT_NE(dec3.get(), nullptr);
+  ASSERT_NE(dec4.get(), nullptr);
+  ASSERT_NE(dec5.get(), nullptr);
+
+  ASSERT_EQ(*dec1.get(), *ptr1.get());
+  ASSERT_EQ(*dec3.get(), *ptr3.get());
+  ASSERT_EQ(*dec4.get(), *ptr1.get());
+  ASSERT_EQ(*dec5.get(), *ptr3.get());
+
+  ASSERT_NE(dec1.get(), dec3.get());
+
+  ASSERT_EQ(dec4.get(), dec1.get());
+  ASSERT_EQ(dec5.get(), dec3.get());
+}
+
+TEST(CustomSerialization, Complex) {
+  std::unique_ptr<std::vector<std::experimental::optional<std::string>>>
+      original = std::make_unique<
+          std::vector<std::experimental::optional<std::string>>>();
+  original.get()->push_back("nandare!");
+  original.get()->push_back(std::experimental::nullopt);
+  original.get()->push_back("hai hai hai");
+
+  slk::Builder builder;
+  slk::Save(original, &builder);
+
+  // clang-format off
+  ASSERT_EQ(builder.size(),
+            sizeof(bool) +
+              sizeof(uint64_t) +
+                sizeof(bool) + sizeof(uint64_t) + (*original.get())[0]->size() +
+                sizeof(bool) +
+                sizeof(bool) + sizeof(uint64_t) + (*original.get())[2]->size());
+  // clang-format on
+
+  std::unique_ptr<std::vector<std::experimental::optional<std::string>>>
+      decoded;
+  ASSERT_EQ(decoded.get(), nullptr);
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_NE(decoded.get(), nullptr);
+  ASSERT_EQ(*original.get(), *decoded.get());
+}
+
+struct Foo {
+  std::string name;
+  std::experimental::optional<int> value;
+};
+
+bool operator==(const Foo &a, const Foo &b) {
+  return a.name == b.name && a.value == b.value;
+}
+
+namespace slk {
+void Save(const Foo &obj, Builder *builder) {
+  Save(obj.name, builder);
+  Save(obj.value, builder);
+}
+
+void Load(Foo *obj, Reader *reader) {
+  Load(&obj->name, reader);
+  Load(&obj->value, reader);
+}
+}  // namespace slk
+
+TEST(CustomSerialization, VectorStruct) {
+  std::vector<Foo> original;
+  original.push_back({"hai hai hai", 5});
+  original.push_back({"nandare!", std::experimental::nullopt});
+
+  slk::Builder builder;
+  slk::Save(original, &builder);
+
+  // clang-format off
+  ASSERT_EQ(builder.size(),
+            sizeof(uint64_t) +
+              sizeof(uint64_t) + original[0].name.size() + sizeof(bool) + sizeof(int) +
+              sizeof(uint64_t) + original[1].name.size() + sizeof(bool));
+  // clang-format on
+
+  std::vector<Foo> decoded;
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load(&decoded, &reader);
+  ASSERT_EQ(original, decoded);
+}
+
+TEST(CustomSerialization, VectorSharedPtr) {
+  std::shared_ptr<std::string> ptr1 = std::make_shared<std::string>("nandare!");
+  std::shared_ptr<std::string> ptr2;
+  std::shared_ptr<std::string> ptr3 =
+      std::make_shared<std::string>("hai hai hai");
+
+  std::vector<std::shared_ptr<std::string>> original;
+  std::vector<std::string *> saved;
+
+  original.push_back(ptr1);
+  original.push_back(ptr2);
+  original.push_back(ptr3);
+  original.push_back(ptr1);
+  original.push_back(ptr3);
+
+  slk::Builder builder;
+  slk::Save<std::shared_ptr<std::string>>(
+      original, &builder, [&saved](const auto &item, auto *builder) {
+        Save(item, builder, &saved);
+      });
+
+  std::vector<std::shared_ptr<std::string>> decoded;
+  std::vector<std::shared_ptr<std::string>> loaded;
+
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load<std::shared_ptr<std::string>>(
+      &decoded, &reader,
+      [&loaded](auto *item, auto *reader) { Load(item, reader, &loaded); });
+
+  ASSERT_EQ(decoded.size(), original.size());
+
+  ASSERT_EQ(saved.size(), 2);
+  ASSERT_EQ(loaded.size(), saved.size());
+
+  ASSERT_EQ(*decoded[0].get(), *ptr1.get());
+  ASSERT_EQ(decoded[1].get(), nullptr);
+  ASSERT_EQ(*decoded[2].get(), *ptr3.get());
+  ASSERT_EQ(*decoded[3].get(), *ptr1.get());
+  ASSERT_EQ(*decoded[4].get(), *ptr3.get());
+
+  ASSERT_NE(decoded[0].get(), decoded[2].get());
+
+  ASSERT_EQ(decoded[3].get(), decoded[0].get());
+  ASSERT_EQ(decoded[4].get(), decoded[2].get());
+}
+
+TEST(CustomSerialization, OptionalSharedPtr) {
+  std::experimental::optional<std::shared_ptr<std::string>> original = std::make_shared<std::string>("nandare!");
+  std::vector<std::string *> saved;
+
+  slk::Builder builder;
+  slk::Save<std::shared_ptr<std::string>>(
+      original, &builder, [&saved](const auto &item, auto *builder) {
+        Save(item, builder, &saved);
+      });
+
+  std::experimental::optional<std::shared_ptr<std::string>> decoded;
+  std::vector<std::shared_ptr<std::string>> loaded;
+
+  slk::Reader reader(builder.data(), builder.size());
+  slk::Load<std::shared_ptr<std::string>>(
+      &decoded, &reader,
+      [&loaded](auto *item, auto *reader) { Load(item, reader, &loaded); });
+
+  ASSERT_NE(decoded, std::experimental::nullopt);
+
+  ASSERT_EQ(saved.size(), 1);
+  ASSERT_EQ(loaded.size(), 1);
+
+  ASSERT_EQ(*decoded->get(), *original->get());
+}