Serialize ast using capnproto

Reviewers: teon.banek, florijan, msantl

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1306

.gitignore

@@ -34,6 +34,6 @@ TAGS
 *.fas
 *.fasl
 
-# Cap'n Proto geenrated files
+# Cap'n Proto generated files
 *.capnp.c++
 *.capnp.h

src/CMakeLists.txt

@@ -103,7 +103,7 @@ function(add_capnp capnp_src_file)
   set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.c++)
   set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h)
   add_custom_command(OUTPUT ${cpp_file} ${h_file}
-    COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file}
+    COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file} -I ${CMAKE_CURRENT_SOURCE_DIR}
     DEPENDS ${capnp_src_file} capnproto-proj
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   # Update *global* generated_capnp_files
@@ -113,6 +113,9 @@ function(add_capnp capnp_src_file)
 endfunction(add_capnp)
 
 add_capnp(query/frontend/semantic/symbol.capnp)
+add_capnp(query/frontend/ast/ast.capnp)
+add_capnp(utils/typed_value.capnp)
+add_capnp(storage/types.capnp)
 
 add_custom_target(generate_capnp DEPENDS ${generated_capnp_files})
 # -----------------------------------------------------------------------------

src/query/frontend/ast/ast.capnp

@@ -0,0 +1,385 @@
+@0xb107d3d6b4b1600b;
+
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("query::capnp");
+
+using Utils = import "/utils/typed_value.capnp";
+using Storage = import "/storage/types.capnp";
+using Symbols = import "/query/frontend/semantic/symbol.capnp";
+
+struct Tree {
+  uid @0 :Int64;
+
+  union {
+    expression @1 :Expression;
+    where @2 :Where;
+    namedExpression @3 :NamedExpression;
+    patternAtom @4 :PatternAtom;
+    pattern @5 :Pattern;
+    clause @6 :Clause;
+    singleQuery @7 :SingleQuery;
+    cypherUnion @8 :CypherUnion;
+    query @9 :Query;
+  }
+}
+
+struct Expression {
+  union {
+    binaryOperator @0 :BinaryOperator;
+    unaryOperator @1 :UnaryOperator;
+    baseLiteral @2 :BaseLiteral;
+    listSlicingOperator @3 :ListSlicingOperator;
+    ifOperator @4 :IfOperator;
+    identifier @5 :Identifier;
+    propertyLookup @6 :PropertyLookup;
+    labelsTest @7 :LabelsTest;
+    function @8 :Function;
+    reduce @9 :Reduce;
+    all @10 :All;
+    single @11 :Single;
+    parameterLookup @12 :ParameterLookup;
+  }
+}
+
+struct Where {
+  expression @0 :Tree;
+}
+
+struct NamedExpression {
+  name @0 :Text;
+  expression @1 :Tree;
+  tokenPosition @2 :Int32;
+}
+
+struct PatternAtom {
+  union {
+    nodeAtom @0 :NodeAtom;
+    edgeAtom @1 :EdgeAtom;
+  }
+  identifier @2 :Tree;
+}
+
+struct Pair(First, Second) {
+  first @0 :First;
+  second @1 :Second;
+}
+
+struct NodeAtom {
+  properties @0 :List(Entry);
+  struct Entry {
+    key @0 :Pair(Text, Storage.Common);
+    value @1 :Tree;
+  }
+
+  labels @1 :List(Storage.Common);
+}
+
+struct EdgeAtom {
+  enum Type {
+    single @0;
+    depthFirst @1;
+    breadthFirst @2;
+    weightedShortestPath @3;
+  }
+  type @0 :Type;
+
+  enum Direction {
+    in @0;
+    out @1;
+    both @2;
+  }
+  direction @1 :Direction;
+
+  properties @2 :List(Entry);
+  struct Entry {
+    key @0 :Pair(Text, Storage.Common);
+    value @1 :Tree;
+  }
+
+  lowerBound @3 :Tree;
+  upperBound @4 :Tree;
+
+  filterLambda @5 :Lambda;
+  weightLambda @6 :Lambda;
+  struct Lambda {
+    innerEdge @0 :Tree;
+    innerNode @1 :Tree;
+    expression @2 :Tree;
+  }
+
+  totalWeight @7 :Tree;
+  edgeTypes @8 :List(Storage.Common);
+}
+
+struct Pattern {
+  identifier @0 :Tree;
+  atoms @1 :List(Tree);
+}
+
+struct Clause {
+  union {
+    create @0 :Create;
+    match @1 :Match;
+    return @2 :Return;
+    with @3 :With;
+    delete @4 :Delete;
+    setProperty @5 :SetProperty;
+    setProperties @6 :SetProperties;
+    setLabels @7 :SetLabels;
+    removeProperty @8 :RemoveProperty;
+    removeLabels @9 :RemoveLabels;
+    merge @10 :Merge;
+    unwind @11 :Unwind;
+    createIndex @12 :CreateIndex;
+  }
+}
+
+struct SingleQuery {
+  clauses @0 :List(Tree);
+}
+
+struct CypherUnion {
+  singleQuery @0 :Tree;
+  distinct @1 :Bool;
+  unionSymbols @2 :List(Symbols.Symbol);
+}
+
+struct Query {
+  singleQuery @0 :Tree;
+  cypherUnions @1 :List(Tree);
+}
+
+struct BinaryOperator {
+  union {
+    orOperator @0 :OrOperator;
+    xorOperator @1 :XorOperator;
+    andOperator @2 :AndOperator;
+    additionOperator @3 :AdditionOperator;
+    subtractionOperator @4 :SubtractionOperator;
+    multiplicationOperator @5 :MultiplicationOperator;
+    divisionOperator @6 :DivisionOperator;
+    modOperator @7 :ModOperator;
+    notEqualOperator @8 :NotEqualOperator;
+    equalOperator @9 :EqualOperator;
+    lessOperator @10 :LessOperator;
+    greaterOperator @11 :GreaterOperator;
+    lessEqualOperator @12 :LessEqualOperator;
+    greaterEqualOperator @13 :GreaterEqualOperator;
+    inListOperator @14 :InListOperator;
+    listMapIndexingOperator @15 :ListMapIndexingOperator;
+    aggregation @16 :Aggregation;
+  }
+  expression1 @17 :Tree;
+  expression2 @18 :Tree;
+}
+
+struct OrOperator {}
+struct XorOperator {}
+struct AndOperator {}
+struct AdditionOperator {}
+struct SubtractionOperator {}
+struct MultiplicationOperator {}
+struct DivisionOperator {}
+struct ModOperator {}
+struct NotEqualOperator {}
+struct EqualOperator {}
+struct LessOperator {}
+struct GreaterOperator {}
+struct LessEqualOperator {}
+struct GreaterEqualOperator {}
+struct InListOperator {}
+struct ListMapIndexingOperator {}
+struct Aggregation {
+  enum Op {
+    count @0;
+    min @1;
+    max @2;
+    sum @3;
+    avg @4 ;
+    collectList @5;
+    collectMap @6;
+  }
+  op @0 :Op;
+}
+
+struct UnaryOperator {
+  union {
+    notOperator @0 :NotOperator;
+    unaryPlusOperator @1 :UnaryPlusOperator;
+    unaryMinusOperator @2 :UnaryMinusOperator;
+    isNullOperator @3 :IsNullOperator;
+  }
+  expression @4 :Tree;
+}
+
+struct NotOperator {}
+struct UnaryPlusOperator {}
+struct UnaryMinusOperator {}
+struct IsNullOperator {}
+
+struct BaseLiteral {
+  union {
+    primitiveLiteral @0 :PrimitiveLiteral;
+    listLiteral @1 :ListLiteral;
+    mapLiteral @2 :MapLiteral;
+  }
+}
+
+struct PrimitiveLiteral {
+  tokenPosition @0 :Int32;
+  value @1 :Utils.TypedValue;
+}
+
+struct ListLiteral {
+  elements @0 :List(Tree);
+}
+
+struct MapLiteral {
+  elements @0 :List(Entry);
+  struct Entry {
+    key @0 :Pair(Text, Storage.Common);
+    value @1 :Tree;
+  }
+}
+
+struct ListSlicingOperator {
+  list @0 :Tree;
+  lowerBound @1 :Tree;
+  upperBound @2 :Tree;
+}
+
+struct IfOperator {
+  condition @0 :Tree;
+  thenExpression @1 :Tree;
+  elseExpression @2 :Tree;
+}
+
+struct Identifier {
+  name @0 :Text;
+  userDeclared @1 :Bool;
+}
+
+struct PropertyLookup {
+  expression @0 :Tree;
+  propertyName @1 :Text;
+  property @2 :Storage.Common;
+}
+
+struct LabelsTest {
+  expression @0 :Tree;
+  labels @1 :List(Storage.Common);
+}
+
+struct Function {
+  functionName @0 :Text;
+  arguments @1 :List(Tree);
+}
+
+struct Reduce {
+  accumulator @0 :Tree;
+  initializer @1 :Tree;
+  identifier @2 :Tree;
+  list @3 :Tree;
+  expression @4 :Tree;
+}
+
+struct All {
+  identifier @0 :Tree;
+  listExpression @1 :Tree;
+  where @2 :Tree;
+}
+
+struct Single {
+  identifier @0 :Tree;
+  listExpression @1 :Tree;
+  where @2 :Tree;
+}
+
+struct ParameterLookup {
+  tokenPosition @0 :Int32;
+}
+
+struct Create {
+  patterns @0 :List(Tree);
+}
+
+struct Match {
+  patterns @0 :List(Tree);
+  where @1 :Tree;
+  optional @2 :Bool;
+}
+
+enum Ordering {
+  asc @0;
+  desc @1;
+}
+
+struct ReturnBody {
+  distinct @0 :Bool;
+  allIdentifiers @1 :Bool;
+  namedExpressions @2 :List(Tree);
+  orderBy @3 :List(Pair);
+
+  struct Pair {
+    ordering @0 :Ordering;
+    expression @1 :Tree;
+  }
+
+  skip @4 :Tree;
+  limit @5 :Tree;
+}
+
+struct Return {
+  returnBody @0 :ReturnBody;
+}
+
+struct With {
+  returnBody @0 :ReturnBody;
+  where @1 :Tree;
+}
+
+struct Delete {
+  detach @0 :Bool;
+  expressions @1 :List(Tree);
+}
+
+struct SetProperty {
+  propertyLookup @0 :Tree;
+  expression @1 :Tree;
+}
+
+struct SetProperties {
+  identifier @0 :Tree;
+  expression @1 :Tree;
+  update @2 :Bool;
+}
+
+struct SetLabels {
+  identifier @0 :Tree;
+  labels @1 :List(Storage.Common);
+}
+
+struct RemoveProperty {
+  propertyLookup @0 :Tree;
+}
+
+struct RemoveLabels {
+  identifier @0 :Tree;
+  labels @1 :List(Storage.Common);
+}
+
+struct Merge {
+  pattern @0 :Tree;
+  onMatch @1 :List(Tree);
+  onCreate @2 :List(Tree);
+}
+
+struct Unwind {
+  namedExpression @0 :Tree;
+}
+
+struct CreateIndex {
+  label @0 :Storage.Common;
+  property @1 :Storage.Common;
+}
+

src/query/frontend/semantic/symbol.hpp

@@ -5,6 +5,8 @@
 #include "boost/serialization/serialization.hpp"
 #include "boost/serialization/string.hpp"
 
+#include "symbol.capnp.h"
+
 namespace query {
 
 class Symbol {
@@ -40,6 +42,60 @@ class Symbol {
   bool user_declared() const { return user_declared_; }
   int token_position() const { return token_position_; }
 
+  void Save(capnp::Symbol::Builder &builder) {
+    builder.setName(name_);
+    builder.setPosition(position_);
+    builder.setUserDeclared(user_declared_);
+    builder.setTokenPosition(token_position_);
+    switch (type_) {
+      case Type::Any:
+        builder.setType(capnp::Symbol::Type::ANY);
+        break;
+      case Type::Edge:
+        builder.setType(capnp::Symbol::Type::EDGE);
+        break;
+      case Type::EdgeList:
+        builder.setType(capnp::Symbol::Type::EDGE_LIST);
+        break;
+      case Type::Number:
+        builder.setType(capnp::Symbol::Type::NUMBER);
+        break;
+      case Type::Path:
+        builder.setType(capnp::Symbol::Type::PATH);
+        break;
+      case Type::Vertex:
+        builder.setType(capnp::Symbol::Type::VERTEX);
+        break;
+    }
+  }
+
+  void Load(const capnp::Symbol::Reader &reader) {
+    name_ = reader.getName();
+    position_ = reader.getPosition();
+    user_declared_ = reader.getUserDeclared();
+    token_position_ = reader.getTokenPosition();
+    switch (reader.getType()) {
+      case capnp::Symbol::Type::ANY:
+        type_ = Type::Any;
+        break;
+      case capnp::Symbol::Type::EDGE:
+        type_ = Type::Edge;
+        break;
+      case capnp::Symbol::Type::EDGE_LIST:
+        type_ = Type::EdgeList;
+        break;
+      case capnp::Symbol::Type::NUMBER:
+        type_ = Type::Number;
+        break;
+      case capnp::Symbol::Type::PATH:
+        type_ = Type::Path;
+        break;
+      case capnp::Symbol::Type::VERTEX:
+        type_ = Type::Vertex;
+        break;
+    }
+  }
+
  private:
   std::string name_;
   int position_;

src/storage/types.capnp

@@ -0,0 +1,17 @@
+@0x8678c6a8817808d9;
+
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("storage::capnp");
+
+struct Common {
+  storage @0 :UInt16;
+  union {
+    label @1 :Label;
+    edgeType @2 :EdgeType;
+    property @3 :Property;
+  }
+}
+
+struct Label {}
+struct EdgeType {}
+struct Property {}

src/storage/types.hpp

@@ -4,6 +4,7 @@
 #include <functional>
 
 #include "boost/serialization/base_object.hpp"
+#include "types.capnp.h"
 
 #include "utils/total_ordering.hpp"
 
@@ -32,6 +33,14 @@ class Common : public utils::TotalOrdering<TSpecificType> {
     size_t operator()(const TSpecificType &t) const { return hash(t.storage_); }
   };
 
+  virtual void Save(capnp::Common::Builder &builder) const {
+    builder.setStorage(storage_);
+  }
+
+  virtual void Load(capnp::Common::Reader &reader) {
+    storage_ = reader.getStorage();
+  }
+
  private:
   friend class boost::serialization::access;
 
@@ -46,6 +55,7 @@ class Common : public utils::TotalOrdering<TSpecificType> {
 class Label : public Common<Label> {
   using Common::Common;
 
+ private:
   friend class boost::serialization::access;
 
   template <class TArchive>
@@ -57,6 +67,7 @@ class Label : public Common<Label> {
 class EdgeType : public Common<EdgeType> {
   using Common::Common;
 
+ private:
   friend class boost::serialization::access;
 
   template <class TArchive>
@@ -68,8 +79,8 @@ class EdgeType : public Common<EdgeType> {
 class Property : public Common<Property> {
   using Common::Common;
 
+ private:
   friend class boost::serialization::access;
-
   template <class TArchive>
   void serialize(TArchive &ar, const unsigned int) {
     ar &boost::serialization::base_object<Common<Property>>(*this);

src/utils/serialization.hpp

@@ -8,6 +8,8 @@
 #include "storage/vertex.hpp"
 #include "utils/exceptions.hpp"
 
+#include "typed_value.capnp.h"
+
 namespace boost::serialization {
 
 namespace {
@@ -60,6 +62,67 @@ void load(TArchive &ar, std::experimental::optional<T> &opt, unsigned int) {
 
 namespace utils {
 
+inline void SaveCapnpTypedValue(const query::TypedValue &value,
+                                query::capnp::TypedValue::Builder &builder) {
+  switch (value.type()) {
+    case query::TypedValue::Type::Null:
+      builder.setNullType();
+      return;
+    case query::TypedValue::Type::Bool:
+      builder.setBool(value.Value<bool>());
+      return;
+    case query::TypedValue::Type::Int:
+      builder.setInteger(value.Value<int64_t>());
+      return;
+    case query::TypedValue::Type::Double:
+      builder.setDouble(value.Value<double>());
+      return;
+    case query::TypedValue::Type::String:
+      builder.setString(value.Value<std::string>());
+      return;
+    case query::TypedValue::Type::List:
+    case query::TypedValue::Type::Map:
+    case query::TypedValue::Type::Vertex:
+    case query::TypedValue::Type::Edge:
+    case query::TypedValue::Type::Path:
+      throw utils::NotYetImplemented("Capnp serialize typed value");
+  }
+}
+
+inline void LoadCapnpTypedValue(query::TypedValue &value,
+                                query::capnp::TypedValue::Reader &reader) {
+  switch (reader.which()) {
+    case query::capnp::TypedValue::BOOL:
+      value = reader.getBool();
+      return;
+    case query::capnp::TypedValue::DOUBLE:
+      value = reader.getDouble();
+      return;
+    // case query::capnp::TypedValue::EDGE:
+    //   // TODO
+    //   return;
+    case query::capnp::TypedValue::INTEGER:
+      value = reader.getInteger();
+      return;
+    case query::capnp::TypedValue::LIST:
+      throw utils::NotYetImplemented("Capnp deserialize typed value");
+    case query::capnp::TypedValue::MAP:
+      throw utils::NotYetImplemented("Capnp deserialize typed value");
+    case query::capnp::TypedValue::NULL_TYPE:
+      value = query::TypedValue::Null;
+      return;
+    // case query::capnp::TypedValue::PATH:
+    //   // TODO
+    //   return;
+    case query::capnp::TypedValue::STRING:
+      value = reader.getString().cStr();
+      return;
+      // case query::capnp::TypedValue::VERTEX:
+      //   // TODO
+      //   return;
+  }
+}
+
 /**
  * Saves the given value into the given Boost archive. The optional
  * `save_graph_element` function is called if the given `value` is a

src/utils/typed_value.capnp

@@ -0,0 +1,25 @@
+@0xd229a9c0f7e55750;
+
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("query::capnp");
+
+
+struct TypedValue {
+  union {
+    nullType @0 :Void;
+    bool @1 :Bool;
+    integer @2 :Int64;
+    double @3 :Float64;
+    string @4 :Text;
+    list @5 :List(TypedValue);
+    map @6 :List(Entry);
+    # TODO vertex accessor
+    # TODO edge accessor
+    # TODO path
+  }
+
+  struct Entry {
+    key @0 :Text;
+    value @1 :TypedValue;
+  }
+}

tests/unit/cypher_main_visitor.cpp

@@ -18,6 +18,10 @@
 #include "query/frontend/stripped.hpp"
 #include "query/typed_value.hpp"
 
+#include "capnp/message.h"
+#include "capnp/serialize-packed.h"
+#include "query/frontend/ast/ast.capnp.h"
+
 namespace {
 
 using namespace query;
@@ -137,12 +141,42 @@ class SerializedAstGenerator : public Base {
   Query *query_;
 };
 
+class CapnpAstGenerator : public Base {
+ public:
+  CapnpAstGenerator(const std::string &query)
+      : Base(query),
+        storage_([&]() {
+          ::frontend::opencypher::Parser parser(query);
+          CypherMainVisitor visitor(context_);
+          visitor.visit(parser.tree());
+
+          ::capnp::MallocMessageBuilder message;
+          {
+            query::capnp::Tree::Builder builder =
+                message.initRoot<query::capnp::Tree>();
+            visitor.query()->Save(builder);
+          }
+
+          AstTreeStorage new_ast;
+          {
+            query::capnp::Tree::Reader reader =
+                message.getRoot<query::capnp::Tree>();
+            new_ast.Load(reader);
+          }
+          return new_ast;
+        }()),
+        query_(storage_.query()) {}
+
+  AstTreeStorage storage_;
+  Query *query_;
+};
+
 template <typename T>
 class CypherMainVisitorTest : public ::testing::Test {};
 
 typedef ::testing::Types<AstGenerator, OriginalAfterCloningAstGenerator,
                          ClonedAstGenerator, CachedAstGenerator,
-                         SerializedAstGenerator>
+                         SerializedAstGenerator, CapnpAstGenerator>
     AstGeneratorTypes;
 TYPED_TEST_CASE(CypherMainVisitorTest, AstGeneratorTypes);
 
@@ -1725,7 +1759,6 @@ TYPED_TEST(CypherMainVisitorTest, MatchWShortestNoFilterReturn) {
   EXPECT_FALSE(shortest->total_weight_->user_declared_);
 }
 
-
 TYPED_TEST(CypherMainVisitorTest, SemanticExceptionOnWShortestLowerBound) {
   ASSERT_THROW(
       TypeParam("MATCH ()-[r *wShortest 10.. (e, n | 42)]-() RETURN r"),