Benchmark Cap'n Proto and boost serialization

Reviewers: florijan, mferencevic, buda, mculinovic

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1273

.gitignore

@@ -33,3 +33,7 @@ TAGS
 # Lisp compiled object code
 *.fas
 *.fasl
+
+# Cap'n Proto geenrated files
+*.capnp.c++
+*.capnp.h

CMakeLists.txt

@@ -150,6 +150,7 @@ include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR})
 include_directories(SYSTEM ${BZIP2_INCLUDE_DIR})
 include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
 include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR})
+include_directories(SYSTEM ${CAPNP_INCLUDE_DIR})
 # -----------------------------------------------------------------------------
 
 # openCypher parser -----------------------------------------------------------

apollo_build.yaml

@@ -17,21 +17,21 @@
     mkdir build_debug
     cd build_debug
     cmake ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Build coverage binaries.
     cd ..
     mkdir build_coverage
     cd build_coverage
     cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1000 make -j$THREADS memgraph__unit
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
 
     # Build release binaries.
     cd ..
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1000 make -j$THREADS memgraph tools memgraph__macro_benchmark memgraph__stress memgraph__manual__card_fraud_generate_snapshot
+    TIMEOUT=1200 make -j$THREADS memgraph tools memgraph__macro_benchmark memgraph__stress memgraph__manual__card_fraud_generate_snapshot
 
     # Generate distributed card fraud dataset.
     cd ../tests/distributed/card_fraud
@@ -47,7 +47,7 @@
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark
+    TIMEOUT=1200 make -j$THREADS memgraph memgraph__macro_benchmark
 
 
 # release build is the default one
@@ -63,21 +63,21 @@
     mkdir build_debug
     cd build_debug
     cmake ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Build coverage binaries.
     cd ..
     mkdir build_coverage
     cd build_coverage
     cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1000 make -j$THREADS memgraph__unit
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
 
     # Build release binaries.
     cd ..
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Create Debian package.
     mkdir output

libs/CMakeLists.txt

@@ -209,3 +209,27 @@ import_external_library(rocksdb STATIC
                      CC=${CMAKE_C_COMPILER}
                      CXX=${CMAKE_CXX_COMPILER}
   INSTALL_COMMAND true)
+
+# Setup Cap'n Proto
+ExternalProject_Add(capnproto-proj
+                    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/configure
+                                      --prefix=${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local
+                                      --enable-shared=no --silent
+                                      CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
+                    BUILD_COMMAND make -j${NPROC} check)
+set(CAPNP_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/include
+    CACHE FILEPATH "Path to capnproto include directory" FORCE)
+set(CAPNP_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libcapnp.a
+    CACHE FILEPATH "Path to capnproto library" FORCE)
+set(KJ_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libkj.a
+    CACHE FILEPATH "Path to kj library (used by capnproto)" FORCE)
+import_library(capnp STATIC ${CAPNP_LIBRARY} capnproto-proj)
+import_library(kj STATIC ${KJ_LIBRARY} capnproto-proj)
+set(CAPNP_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnp
+    CACHE FILEPATH "Path to capnproto executable" FORCE)
+set(CAPNP_CXX_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnpc-c++
+    CACHE FILEPATH "Path to capnproto c++ plugin executable" FORCE)
+mark_as_advanced(CAPNP_INCLUDE_DIR CAPNP_LIBRARY KJ_LIBRARY CAPNP_EXE CAPNP_CXX_EXE)

libs/setup.sh

@@ -120,3 +120,10 @@ clone git://deps.memgraph.io/zlib.git zlib $zlib_tag
 
 rocksdb_tag="dbd8fa09b823826dd2a30bc119dad7a6fa9a4c6d" # v5.11.3 Mar 12, 2018
 clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag
+
+# Cap'n Proto serialization (and RPC) lib
+wget -nv http://deps.memgraph.io/capnproto-c++-0.6.1.tar.gz -O capnproto.tar.gz
+tar -xzf capnproto.tar.gz
+rm -rf capnproto
+mv capnproto-c++-0.6.1 capnproto
+rm capnproto.tar.gz

src/CMakeLists.txt

@@ -95,13 +95,33 @@ function(add_lcp lcp_file)
 endfunction(add_lcp)
 
 add_custom_target(generate_lcp DEPENDS ${generated_lcp_files})
+
+# Use this function to add each capnp file to generation. This way each file is
+# standalone and we avoid recompiling everything.
+# NOTE: memgraph_src_files and generated_capnp_files are globally updated.
+function(add_capnp capnp_src_file)
+  set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.c++)
+  set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h)
+  add_custom_command(OUTPUT ${cpp_file} ${h_file}
+    COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file}
+    DEPENDS ${capnp_src_file} capnproto-proj
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  # Update *global* generated_capnp_files
+  set(generated_capnp_files ${generated_capnp_files} ${cpp_file} ${h_file} PARENT_SCOPE)
+  # Update *global* memgraph_src_files
+  set(memgraph_src_files ${memgraph_src_files} ${cpp_file} PARENT_SCOPE)
+endfunction(add_capnp)
+
+add_capnp(query/frontend/semantic/symbol.capnp)
+
+add_custom_target(generate_capnp DEPENDS ${generated_capnp_files})
 # -----------------------------------------------------------------------------
 
 string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
 
 # memgraph_lib depend on these libraries
 set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools
-    antlr_opencypher_parser_lib dl glog gflags
+    antlr_opencypher_parser_lib dl glog gflags capnp kj
     ${Boost_IOSTREAMS_LIBRARY_RELEASE}
     ${Boost_SERIALIZATION_LIBRARY_RELEASE})
 
@@ -120,6 +140,7 @@ add_library(memgraph_lib STATIC ${memgraph_src_files})
 target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS})
 add_dependencies(memgraph_lib generate_opencypher_parser)
 add_dependencies(memgraph_lib generate_lcp)
+add_dependencies(memgraph_lib generate_capnp)
 
 # STATIC library used to store key-value pairs
 # TODO: Create a utils lib to link with, and remove utils/file.cpp.

src/query/frontend/semantic/symbol.capnp

@@ -0,0 +1,21 @@
+@0x93c1dcee84e93b76;
+
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("query::capnp");
+
+struct Symbol {
+  enum Type {
+    any @0;
+    vertex @1;
+    edge @2;
+    path @3;
+    number @4;
+    edgeList @5;
+  }
+
+  name @0 :Text;
+  position @1 :Int32;
+  type @2 :Type;
+  userDeclared @3 :Bool;
+  tokenPosition @4 :Int32;
+}

tests/benchmark/serialization.cpp

@@ -0,0 +1,143 @@
+#include <sstream>
+#include <string>
+
+#include <benchmark/benchmark.h>
+
+#include "boost/archive/binary_iarchive.hpp"
+#include "boost/archive/binary_oarchive.hpp"
+#include "boost/serialization/vector.hpp"
+
+#include <capnp/serialize.h>
+#include <kj/std/iostream.h>
+
+#include "query/frontend/semantic/symbol.capnp.h"
+#include "query/frontend/semantic/symbol.hpp"
+
+class SymbolVectorFixture : public benchmark::Fixture {
+ protected:
+  std::vector<query::Symbol> symbols_;
+
+  void SetUp(const benchmark::State &state) override {
+    using Type = ::query::Symbol::Type;
+    std::vector<Type> types{Type::Any,  Type::Vertex, Type::Edge,
+                            Type::Path, Type::Number, Type::EdgeList};
+    symbols_.reserve(state.range(0));
+    for (int i = 0; i < state.range(0); ++i) {
+      std::string name = "Symbol " + std::to_string(i);
+      bool user_declared = i % 2;
+      auto type = types[i % types.size()];
+      symbols_.emplace_back(name, i, user_declared, type, i);
+    }
+  }
+
+  void TearDown(const benchmark::State &) override { symbols_.clear(); }
+};
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, BoostSerial)(benchmark::State &state) {
+  while (state.KeepRunning()) {
+    std::stringstream stream(std::ios_base::out | std::ios_base::binary);
+    {
+      boost::archive::binary_oarchive archive(stream);
+      archive << symbols_;
+    }
+  }
+}
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, BoostDeserial)
+(benchmark::State &state) {
+  auto serialize = [this]() {
+    std::stringstream stream(std::ios_base::in | std::ios_base::out |
+                             std::ios_base::binary);
+    {
+      boost::archive::binary_oarchive archive(stream);
+      archive << symbols_;
+    }
+    return stream;
+  };
+  while (state.KeepRunning()) {
+    state.PauseTiming();
+    auto stream = serialize();
+    state.ResumeTiming();
+    std::vector<query::Symbol> symbols;
+    {
+      boost::archive::binary_iarchive archive(stream);
+      archive >> symbols;
+    }
+  }
+}
+
+void SymbolVectorToCapnpMessage(const std::vector<query::Symbol> &symbols,
+                                capnp::MessageBuilder &message) {
+  auto symbols_builder =
+      message.initRoot<capnp::List<query::capnp::Symbol>>(symbols.size());
+  for (int i = 0; i < symbols.size(); ++i) {
+    const auto &sym = symbols[i];
+    query::capnp::Symbol::Builder sym_builder = symbols_builder[i];
+    sym_builder.setName(sym.name());
+    sym_builder.setPosition(sym.position());
+    sym_builder.setType(query::capnp::Symbol::Type::ANY);
+    sym_builder.setUserDeclared(sym.user_declared());
+    sym_builder.setTokenPosition(sym.token_position());
+  }
+}
+
+std::stringstream SerializeCapnpSymbolVector(
+    const std::vector<query::Symbol> &symbols) {
+  std::stringstream stream(std::ios_base::in | std::ios_base::out |
+                           std::ios_base::binary);
+  {
+    capnp::MallocMessageBuilder message;
+    SymbolVectorToCapnpMessage(symbols, message);
+    kj::std::StdOutputStream std_stream(stream);
+    kj::BufferedOutputStreamWrapper buffered_stream(std_stream);
+    writeMessage(buffered_stream, message);
+  }
+  return stream;
+}
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, CapnpSerial)(benchmark::State &state) {
+  while (state.KeepRunning()) {
+    SerializeCapnpSymbolVector(symbols_);
+  }
+}
+
+BENCHMARK_DEFINE_F(SymbolVectorFixture, CapnpDeserial)
+(benchmark::State &state) {
+  while (state.KeepRunning()) {
+    state.PauseTiming();
+    auto stream = SerializeCapnpSymbolVector(symbols_);
+    state.ResumeTiming();
+    kj::std::StdInputStream std_stream(stream);
+    capnp::InputStreamMessageReader message(std_stream);
+    auto symbols_reader = message.getRoot<capnp::List<query::capnp::Symbol>>();
+    std::vector<query::Symbol> symbols;
+    symbols.reserve(symbols_reader.size());
+    for (const auto &sym : symbols_reader) {
+      symbols.emplace_back(sym.getName().cStr(), sym.getPosition(),
+                           sym.getUserDeclared(), query::Symbol::Type::Any,
+                           sym.getTokenPosition());
+    }
+  }
+}
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, BoostSerial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, CapnpSerial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, BoostDeserial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
+BENCHMARK_REGISTER_F(SymbolVectorFixture, CapnpDeserial)
+    ->RangeMultiplier(4)
+    ->Range(4, 1 << 12)
+    ->Unit(benchmark::kNanosecond);
+
+BENCHMARK_MAIN();