From e56ed0accedaa7772ebfac88ea2fe7027e1bdcd5 Mon Sep 17 00:00:00 2001 From: Teon Banek Date: Wed, 16 May 2018 13:48:56 +0200 Subject: [PATCH] Add generating Capnp schema in LCP Summary: Add additional structs and functions for handling C++ meta information. Add capnp-file and capnp-id arguments to lcp:process-file. Generate cpp along with hpp and capnp in lcp. Wrap LogicalOperator base class in lcp:define-class. Modify logical operators for capnp serialization. Add query/common.capnp. Reviewers: mculinovic, buda, mtomic, msantl, ipaljak, dgleich, mferencevic Reviewed By: msantl Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D1391 --- .gitignore | 4 +- src/CMakeLists.txt | 68 +- src/lisp/lcp.lisp | 1081 +++++++++++++++++++++--- src/query/common.capnp | 15 + src/query/common.cpp | 23 + src/query/common.hpp | 7 +- src/query/frontend/ast/ast.cpp | 43 +- src/query/frontend/semantic/symbol.hpp | 22 +- src/query/plan/operator.lcp | 779 ++++++++++++----- src/storage/types.hpp | 6 +- src/utils/serialization.capnp | 13 +- src/utils/serialization.hpp | 10 +- tests/unit/cypher_main_visitor.cpp | 1 - tests/unit/query_planner.cpp | 148 +++- tests/unit/serialization.cpp | 95 +-- tools/lcp | 28 +- 16 files changed, 1917 insertions(+), 426 deletions(-) create mode 100644 src/query/common.capnp diff --git a/.gitignore b/.gitignore index 18fe12942..d56046f8a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,5 +38,7 @@ TAGS *.capnp.c++ *.capnp.h -# LCP generated C++ files +# LCP generated C++ & Cap'n Proto files src/query/plan/operator.hpp +src/query/plan/operator.lcp.cpp +src/query/plan/operator.capnp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87a5ae3a0..e169c062b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -73,30 +73,6 @@ set(memgraph_src_files ) # ----------------------------------------------------------------------------- -# Lisp C++ Preprocessing - -set(lcp_exe ${CMAKE_SOURCE_DIR}/tools/lcp) -set(lcp_src_files lisp/lcp.lisp ${lcp_exe}) - -# Use this function to add each lcp file to generation. This way each file is -# standalone and we avoid recompiling everything. -# NOTE: Only .hpp files are generated from .lcp, so there's no need to update memgraph_src_files. -# NOTE: generated_lcp_files are globally updated. -function(add_lcp lcp_file) - string(REGEX REPLACE "\.lcp$" ".hpp" h_file - "${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}") - add_custom_command(OUTPUT ${h_file} - COMMAND ${lcp_exe} ${lcp_file} > ${h_file} - DEPENDS ${lcp_file} ${lcp_src_files} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - # Update *global* generated_lcp_files - set(generated_lcp_files ${generated_lcp_files} ${h_file} PARENT_SCOPE) -endfunction(add_lcp) - -add_lcp(query/plan/operator.lcp) - -add_custom_target(generate_lcp DEPENDS ${generated_lcp_files}) - # Use this function to add each capnp file to generation. This way each file is # standalone and we avoid recompiling everything. # NOTE: memgraph_src_files and generated_capnp_files are globally updated. @@ -105,7 +81,7 @@ function(add_capnp capnp_src_file) set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h) add_custom_command(OUTPUT ${cpp_file} ${h_file} COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file} -I ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${capnp_src_file} capnproto-proj + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file} capnproto-proj WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) # Update *global* generated_capnp_files set(generated_capnp_files ${generated_capnp_files} ${cpp_file} ${h_file} PARENT_SCOPE) @@ -113,12 +89,52 @@ function(add_capnp capnp_src_file) set(memgraph_src_files ${memgraph_src_files} ${cpp_file} PARENT_SCOPE) endfunction(add_capnp) +# Lisp C++ Preprocessing + +set(lcp_exe ${CMAKE_SOURCE_DIR}/tools/lcp) +set(lcp_src_files lisp/lcp.lisp ${lcp_exe}) + +# Use this function to add each lcp file to generation. This way each file is +# standalone and we avoid recompiling everything. +# NOTE: memgraph_src_files and generated_lcp_files are globally updated. +function(add_lcp lcp_file) + set(options CAPNP_SCHEMA) + cmake_parse_arguments(KW "${options}" "" "" ${ARGN}) + string(REGEX REPLACE "\.lcp$" ".hpp" h_file + "${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}") + if (KW_CAPNP_SCHEMA) + string(REGEX REPLACE "\.lcp$" ".capnp" capnp_file + "${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}") + set(capnp_id_command ${CAPNP_EXE}) + set(capnp_depend capnproto-proj) + set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}.cpp) + # Update *global* memgraph_src_files + set(memgraph_src_files ${memgraph_src_files} ${cpp_file} PARENT_SCOPE) + endif() + add_custom_command(OUTPUT ${h_file} ${cpp_file} ${capnp_file} + COMMAND ${lcp_exe} ${lcp_file} ${capnp_id_command} + VERBATIM + DEPENDS ${lcp_file} ${lcp_src_files} ${capnp_depend} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + # Update *global* generated_lcp_files + set(generated_lcp_files ${generated_lcp_files} ${h_file} ${cpp_file} ${capnp_file} PARENT_SCOPE) +endfunction(add_lcp) + +add_lcp(query/plan/operator.lcp CAPNP_SCHEMA) +add_capnp(query/plan/operator.capnp) + +add_custom_target(generate_lcp DEPENDS ${generated_lcp_files}) + +# Registering capnp must come after registering lcp files. + add_capnp(query/frontend/semantic/symbol.capnp) add_capnp(query/frontend/ast/ast.capnp) add_capnp(utils/serialization.capnp) add_capnp(storage/types.capnp) +add_capnp(query/common.capnp) + +add_custom_target(generate_capnp DEPENDS generate_lcp ${generated_capnp_files}) -add_custom_target(generate_capnp DEPENDS ${generated_capnp_files}) # ----------------------------------------------------------------------------- string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) diff --git a/src/lisp/lcp.lisp b/src/lisp/lcp.lisp index 8e2f3fc53..54f097aca 100644 --- a/src/lisp/lcp.lisp +++ b/src/lisp/lcp.lisp @@ -2,6 +2,12 @@ (:use #:cl) (:export #:define-class #:define-struct + #:define-enum + #:namespace + #:pop-namespace + #:capnp-namespace + #:capnp-import + #:capnp-type-conversion #:process-file)) (in-package #:lcp) @@ -35,7 +41,7 @@ (read-char stream t nil t) ;; consume { (let ((form (let ((*readtable* (copy-readtable))) ;; Read form to } - (set-syntax-from-char #\} #\)) + (set-macro-character #\} (get-macro-character #\))) (read-delimited-list #\} stream t)))) (unless (and (not (null form)) (null (cdr form)) (symbolp (car form))) (error "Expected a variable inside ${..}, got ~A" form)) @@ -75,6 +81,86 @@ (defun cpp-primitive-type-p (type) (member type '(:bool :int :int32_t :int64_t :uint :uint32_t :uint64_t :float :double))) +;; TODO: Rename this to cpp-type and set it as the base class/struct for all +;; the other cpp type metainformation (`CPP-CLASS', `CPP-ENUM' ...) +(defstruct cpp-type-decl + namespace + name + type-args) + +(defun parse-cpp-type-declaration (type-decl) + "Parse C++ type from TYPE-DECL string and return CPP-TYPE-DECL. + +For example: + +::std::pair, double>, char> + +produces: + +;; (cpp-type-decl +;; :name pair +;; :type-args ((cpp-type-decl +;; :name MyClass +;; :type-args ((cpp-type-decl :name function +;; :type-args (cpp-type-decl :name void(int, bool))) +;; (cpp-type-decl :name double))) +;; (cpp-type-decl :name char)))" + (declare (type string type-decl)) + ;; C++ type can be declared as follows: + ;; namespace::namespace::type * + ;; |^^^^^^^^^^^^^^^^^^^^| |^^^^^^^^^^^^^^^^^^| | optional + ;; optional optional + ;; type-args in template are recursively parsed + ;; C++ may contain dependent names with 'typename' keyword, these aren't + ;; supported here. + ;; TODO: Add support for raw pointers as if they are templated + ;; type. I.e. 'char *' should generate + ;; (cpp-type-decl :name * :type-args (cpp-type-decl :name "char")) + (when (search "typename" type-decl) + (error "'typename' not supported in '~A'" type-decl)) + (destructuring-bind (full-name &optional template) + (cl-ppcre:split "<" type-decl :limit 2) + (let* ((namespace-split (cl-ppcre:split "::" full-name)) + (name (car (last namespace-split))) + type-args) + (when template + ;; template ends with '>' character + (let ((arg-start 0)) + (cl-ppcre:do-scans (match-start match-end reg-starts reg-ends + "[a-zA-Z0-9_:<>()]+[,>]" template) + (flet ((matchedp (open-char close-char) + "Return T if the TEMPLATE[ARG-START:MATCH-END] contains + matched OPEN-CHAR and CLOSE-CHAR." + (= (count open-char template :start arg-start :end match-end) + (count close-char template :start arg-start :end match-end)))) + (when (or (= match-end (length template)) ;; we are at the end + (and (matchedp #\< #\>) (matchedp #\( #\)))) + (push (parse-cpp-type-declaration + ;; take the arg and omit final [,>] + (subseq template arg-start (1- match-end))) + type-args) + (setf arg-start (1+ match-end))))))) + (make-cpp-type-decl :namespace (when (cdr namespace-split) + (butlast namespace-split)) + :name name + :type-args (reverse type-args))))) + +(defun string<-cpp-type-decl (type-decl) + (declare (type cpp-type-decl type-decl)) + (with-output-to-string (s) + (format s "~{~A::~}" (cpp-type-decl-namespace type-decl)) + (write-string (cpp-type-decl-name type-decl) s) + (when (cpp-type-decl-type-args type-decl) + (format s "<~{~A~^, ~}>" + (mapcar #'string<-cpp-type-decl (cpp-type-decl-type-args type-decl)))))) + +(defstruct cpp-enum + "Meta information on a C++ enum." + (symbol nil :type symbol :read-only t) + (documentation nil :type (or null string) :read-only t) + (values nil :read-only t) + (enclosing-class nil :type (or null symbol) :read-only t)) + (defstruct cpp-member "Meta information on a C++ class (or struct) member variable." (symbol nil :type symbol :read-only t) @@ -87,7 +173,25 @@ ;; Custom saving and loading code. May be a function which takes 2 ;; args: (archive member-name) and needs to return C++ code. (save-fun nil :type (or null string raw-cpp function) :read-only t) - (load-fun nil :type (or null string raw-cpp function) :read-only t)) + (load-fun nil :type (or null string raw-cpp function) :read-only t) + (capnp-type nil :type (or null string) :read-only t) + (capnp-init t :type boolean :read-only t) + (capnp-save nil :type (or null function) :read-only t) + (capnp-load nil :type (or null function) :read-only t)) + +(defstruct capnp-opts + "Cap'n Proto serialization options for C++ class." + ;; BASE is T if the class should be treated as a base class for capnp, even + ;; though it may have parents. + (base nil :type boolean :read-only t) + ;; Extra arguments to the generated save function. List of (name cpp-type). + (save-args nil :read-only t) + (load-args nil :read-only t) + ;; Explicit instantiation of template to generate schema with enum. + (type-args nil :read-only t) + ;; In case of multiple inheritance, list of classes which should be handled + ;; as a composition. + (inherit-compose nil :read-only t)) (defstruct cpp-class "Meta information on a C++ class (or struct)." @@ -101,7 +205,39 @@ ;; single element. (public nil :read-only t) (protected nil :read-only t) - (private nil)) + (private nil) + (capnp-opts nil :type (or null capnp-opts) :read-only t) + (namespace nil :read-only t) + (inner-types nil) + (enclosing-class nil :type (or null symbol))) + +(defun cpp-type-name (name) + "Get C++ style type name from NAME as a string." + (typecase name + (cpp-primitive-type (string-downcase (string name))) + (symbol (remove #\- (string-capitalize (string name)))) + (string name) + (otherwise (error "Unkown conversion to C++ type for ~S" (type-of name))))) + +(defvar *cpp-classes* nil "List of defined classes from LCP file") + +(defun find-cpp-class (cpp-class-name) + "Find CPP-CLASS in *CPP-CLASSES* by CPP-CLASS-NAME" + (declare (type (or symbol string) cpp-class-name)) + (if (stringp cpp-class-name) + (find cpp-class-name *cpp-classes* + :key (lambda (class) (cpp-type-name (cpp-class-name class))) + :test #'string=) + (find cpp-class-name *cpp-classes* :key #'cpp-class-name))) + +(defun direct-subclasses-of (cpp-class) + "Find direct subclasses of CPP-CLASS from *CPP-CLASSES*" + (declare (type (or symbol cpp-class) cpp-class)) + (let ((name (if (symbolp cpp-class) cpp-class (cpp-class-name cpp-class)))) + (reverse ;; reverse to get them in definition order + (remove-if (lambda (subclass) + (not (member name (cpp-class-super-classes subclass)))) + *cpp-classes*)))) (defun cpp-documentation (documentation) "Convert DOCUMENTATION to Doxygen style string." @@ -112,34 +248,46 @@ documentation (format nil "~%/// ")))) +(defun cpp-variable-name (symbol) + "Get C++ style name of SYMBOL as a string." + (declare (type (or string symbol) symbol)) + (cl-ppcre:regex-replace-all "-" (string-downcase symbol) "_")) + +(defun cpp-constant-name (symbol) + "Get C++ style constant name of SYMBOL as a string. This is like +`CPP-VARIABLE-NAME' but upcased." + (declare (type (or string symbol) symbol)) + (cl-ppcre:regex-replace-all "-" (string-upcase symbol) "_")) + (defun cpp-member-name (cpp-member &key struct) "Get C++ style name of the `CPP-MEMBER' as a string." (declare (type cpp-member cpp-member) (type boolean struct)) - (let ((cpp-name (cl-ppcre:regex-replace-all - "-" (string-downcase (cpp-member-symbol cpp-member)) "_"))) - (if struct - cpp-name - (format nil "~A_" cpp-name)))) + (let ((cpp-name (cpp-variable-name (cpp-member-symbol cpp-member)))) + (if struct cpp-name (format nil "~A_" cpp-name)))) + +(defun cpp-enum-definition (cpp-enum) + "Get C++ style `CPP-ENUM' definition as a string." + (declare (type cpp-enum cpp-enum)) + (with-output-to-string (s) + (when (cpp-enum-documentation cpp-enum) + (write-line (cpp-documentation (cpp-enum-documentation cpp-enum)) s)) + (format s "enum class ~A {~%" (cpp-type-name (cpp-enum-symbol cpp-enum))) + (format s "~{ ~A~^,~%~}~%" (mapcar #'cpp-constant-name (cpp-enum-values cpp-enum))) + (write-line "};" s))) (defun cpp-member-declaration (cpp-member &key struct) "Get C++ style `CPP-MEMBER' declaration as a string." (declare (type cpp-member cpp-member) (type boolean struct)) - (flet ((cpp-type-name () - (cond - ((stringp (cpp-member-type cpp-member)) - (cpp-member-type cpp-member)) - ((keywordp (cpp-member-type cpp-member)) - (string-downcase (string (cpp-member-type cpp-member)))) - (t (error "Unknown conversion to C++ type for ~S" (type-of (cpp-member-type cpp-member))))))) + (let ((type-name (cpp-type-name (cpp-member-type cpp-member)))) (with-output-to-string (s) (when (cpp-member-documentation cpp-member) (write-line (cpp-documentation (cpp-member-documentation cpp-member)) s)) (if (cpp-member-initval cpp-member) - (format s "~A ~A{~A};" (cpp-type-name) + (format s "~A ~A{~A};" type-name (cpp-member-name cpp-member :struct struct) (cpp-member-initval cpp-member)) - (format s "~A ~A;" (cpp-type-name) (cpp-member-name cpp-member :struct struct)))))) + (format s "~A ~A;" type-name (cpp-member-name cpp-member :struct struct)))))) (defun cpp-member-reader-definition (cpp-member) "Get C++ style `CPP-MEMBER' getter (reader) function." @@ -148,9 +296,11 @@ (format nil "auto ~A() const { return ~A; }" (cpp-member-name cpp-member :struct t) (cpp-member-name cpp-member)) (format nil "const auto &~A() const { return ~A; }" (cpp-member-name cpp-member :struct t) (cpp-member-name cpp-member)))) -(defun cpp-type-name (symbol-name) - "Get C++ style type name from lisp SYMBOL-NAME as a string." - (remove #\- (string-capitalize (string symbol-name)))) +(defun cpp-template (type-params &optional stream) + "Generate C++ template declaration from provided TYPE-PARAMS. If STREAM is +NIL, returns a string." + (format stream "template <~{class ~A~^,~^ ~}>" + (mapcar #'cpp-type-name type-params))) (defun cpp-class-definition (cpp-class) "Get C++ definition of the CPP-CLASS as a string." @@ -165,8 +315,7 @@ (when (cpp-class-documentation cpp-class) (write-line (cpp-documentation (cpp-class-documentation cpp-class)) s)) (when (cpp-class-type-params cpp-class) - (format s "template <~{class ~A~^,~^ ~}>~%" - (mapcar #'cpp-type-name (cpp-class-type-params cpp-class)))) + (cpp-template (cpp-class-type-params cpp-class) s)) (if (cpp-class-structp cpp-class) (write-string "struct " s) (write-string "class " s)) @@ -184,6 +333,13 @@ (format s "~{~%~A~}~%" (mapcar #'cpp-member-reader-definition reader-members)) (format s "~{ ~%~A~}~%" (mapcar #'member-declaration (cpp-class-members-scoped :public))))) + (when (cpp-class-capnp-opts cpp-class) + (let ((save (capnp-save-declaration cpp-class)) + (construct (capnp-construct-declaration cpp-class)) + (load (capnp-load-declaration cpp-class))) + (when save (format s " ~A;~%" save)) + (when construct (format s " ~A;~%" construct)) + (when load (format s " ~A;~%" load)))) (when (or (cpp-class-protected cpp-class) (cpp-class-members-scoped :protected)) (write-line " protected:" s) (format s "~{~A~%~}" (mapcar #'cpp-code (cpp-class-protected cpp-class))) @@ -196,11 +352,61 @@ (mapcar #'member-declaration (cpp-class-members-scoped :private)))) (write-line "};" s)))) +(defun cpp-class-full-name (class &key (type-args t)) + "Return the fully namespaced name of given CLASS." + (declare (type cpp-class class)) + (flet ((enclosing-classes (cpp-class) + (declare (type cpp-class cpp-class)) + (let (enclosing) + (loop + for class = cpp-class + then (find-cpp-class (cpp-class-enclosing-class class)) + while class + do (push (cpp-type-name (cpp-class-name class)) enclosing)) + enclosing))) + (let* ((full-name (format nil "~{~A~^::~}" (enclosing-classes class))) + (type-args (if (or (not type-args) (not (cpp-class-type-params class))) + "" + (format nil "<~{~A~^, ~}>" (mapcar #'cpp-type-name (cpp-class-type-params class)))))) + (concatenate 'string full-name type-args)))) + +(defun cpp-method-declaration (class method-name + &key args (returns "void") (inline t) static + virtual const override) + "Generate a C++ method declaration as a string for the given METHOD-NAME on +CLASS. ARGS is a list of (variable type) arguments to method. RETURNS is the +return type of the function. When INLINE is set to NIL, generates a +declaration to be used outside of class definition. Remaining keys are flags +which generate the corresponding C++ keywords." + (declare (type cpp-class class) + (type string method-name)) + (let* ((type-params (cpp-class-type-params class)) + (template (if (or inline (not type-params)) "" (cpp-template type-params))) + (static/virtual (cond + ((and inline static) "static") + ((and inline virtual) "virtual") + (t ""))) + (namespace (if inline "" (format nil "~A::" (cpp-class-full-name class)))) + (args (format nil "~{~A~^, ~}" + (mapcar (lambda (name-and-type) + (format nil "~A ~A" + (cpp-type-name (second name-and-type)) + (cpp-variable-name (first name-and-type)))) + args))) + (const (if const "const" "")) + (override (if (and override inline) "override" ""))) + (raw-cpp-string + #>cpp + ${template} ${static/virtual} + ${returns} ${namespace}${method-name}(${args}) ${const} ${override} + cpp<#))) + (defun cpp-code (cpp) "Get a C++ string from given CPP meta information." (typecase cpp (raw-cpp (raw-cpp-string cpp)) (cpp-class (cpp-class-definition cpp)) + (cpp-enum (cpp-enum-definition cpp)) (string cpp) (null "") (otherwise (error "Unknown conversion to C++ for ~S" (type-of cpp))))) @@ -211,35 +417,6 @@ until (or (not char) (and stop-position (> pos stop-position))) when (char= #\Newline char) count it)) -(defun process-file (filepath &key out-stream) - "Process a LCP file from FILEPATH and send the output to OUT-STREAM." - (flet ((process-to (out) - (with-open-file (in-stream filepath) - (let ((stream-pos 0)) - (handler-case - (loop for form = (read-preserving-whitespace in-stream nil 'eof) - until (eq form 'eof) - for res = (handler-case (eval form) - (error (err) - (file-position in-stream 0) ;; start of stream - (error "~%~A:~A: error:~%~%~A~%~%in:~%~%~A" - (uiop:native-namestring filepath) - (count-newlines in-stream :stop-position (1+ stream-pos)) - err form))) - do (setf stream-pos (file-position in-stream)) - when (typep res '(or raw-cpp cpp-class)) - do (write-line (cpp-code res) out)) - (end-of-file () - (file-position in-stream 0) ;; start of stream - (error "~%~A:~A:error: READ error, did you forget a closing ')'?" - (uiop:native-namestring filepath) - (count-newlines in-stream - :stop-position (1+ stream-pos))))))))) - (if out-stream - (process-to out-stream) - (with-output-to-string (string) - (process-to string))))) - (defun boost-serialization (cpp-class) "Add boost serialization code to `CPP-CLASS'." (labels ((get-serialize-code (member-name serialize-fun) @@ -306,21 +483,600 @@ (mapcar #'load-member members)))) (list #>cpp } cpp<#))))) +;;; Cap'n Proto schema and C++ serialization code generation + +;;; Schema generation +;;; +;;; The basic algorthm should work as follows. +;;; 1) C++ class (or struct) is converted to the same named Capnp struct. +;;; 2) C++ class members are converted to camelCased members of Capnp struct. +;;; a) Primitive C++ types are mapped to primitive Capnp types. +;;; b) C++ std types are converted to our Capnp wrappers found in +;;; utils/serialization.capnp. This process is hardcoded. +;;; c) For other composite types, we assume that a Capnp struct exists +;;; with the same PascalCase name as the *top-most* base class. +;;; d) The user may provide a :CAPNP-TYPE string, which overrides our +;;; decision making. Alternatively, a user may register the conversion +;;; with `CAPNP-TYPE-CONVERSION'. +;;; 3) Handle C++ inheritance by checking direct subclasses of C++ class. +;;; * Inheritance can be modeled with a union or composition. +;;; +;;; Since with Capnp we cannot use the pointer casting trick from C +;;; when modeling inheritance with composition, we are only left with +;;; union. This is an ok solution for single inheritance trees, but C++ +;;; allows for multiple inheritance. This poses a problem. +;;; +;;; Luckily, most of our use cases of multiple inheritance are only for +;;; mixin classes to reuse some functionality. This allows us to model +;;; multiple inheritance via composition. These classes need to be +;;; marked as such for Capnp. Obviously, this precludes serialization +;;; of pointers to mixin classes, but we should deal with that in C++ +;;; code on a case by case basis. +;;; +;;; The algorithm is then the following. +;;; a) If C++ class is the only parent of the direct sublcass, add the +;;; direct subclass to union. +;;; b) If C++ class isn't the only parent (multiple inheritance), then +;;; check whether our class marked to be composed in the derived. If +;;; yes, don't generate the union for direct subclass. Otherwise, +;;; behave as 3.a) +;;; 4) Handle C++ template parameters of a class +;;; Currently we require that explicit instantiations be listed in +;;; TYPE-ARGS of `CAPNP-OPTS'. These arguments are then used to generate +;;; a union, similarly to how inheritance is handled. This approach does +;;; not support inheriting from template classes. + +(defun capnp-union-subclasses (cpp-class) + "Get direct subclasses of CPP-CLASS which should be modeled as a union in +Cap'n Proto schema." + (declare (type (or symbol cpp-class) cpp-class)) + (let ((class-name (if (symbolp cpp-class) cpp-class (cpp-class-name cpp-class)))) + (remove-if (lambda (subclass) + (member class-name + (capnp-opts-inherit-compose (cpp-class-capnp-opts subclass)))) + (direct-subclasses-of cpp-class)))) + +(defun capnp-union-and-compose-parents (cpp-class) + "Get direct parents of CPP-CLASS that model the inheritance via union. The +secondary value contains parents which are modeled as being composed inside +CPP-CLASS." + (declare (type (or symbol cpp-class) cpp-class)) + (let* ((class (if (symbolp cpp-class) (find-cpp-class cpp-class) cpp-class)) + (capnp-opts (cpp-class-capnp-opts class)) + union compose) + (dolist (parent (cpp-class-super-classes class)) + (if (member parent (capnp-opts-inherit-compose capnp-opts)) + (push parent compose) + (push parent union))) + (values union compose))) + +(defun capnp-union-parents-rec (cpp-class) + "Return a list of all parent clases recursively for CPP-CLASS that should be +encoded as union inheritance in Cap'n Proto." + (declare (type cpp-class cpp-class)) + (labels ((capnp-base-p (class) + (declare (type cpp-class class)) + (let ((opts (cpp-class-capnp-opts class))) + (and opts (capnp-opts-base opts)))) + (rec (class) + (declare (type cpp-class class)) + (cons (cpp-class-name class) + ;; Continue to supers only if this isn't marked as capnp base class. + (when (and (not (capnp-base-p class)) + (capnp-union-and-compose-parents class)) + (let ((first-parent (find-cpp-class + (first (capnp-union-and-compose-parents class))))) + (if first-parent + (rec first-parent) + (list (first (capnp-union-and-compose-parents class))))))))) + (cdr (rec cpp-class)))) + +(defvar *capnp-type-converters* nil + "Pairs of (cpp-type capnp-type) which map the conversion of C++ types to + Cap'n Proto types.") + +(defun capnp-type-conversion (cpp-type capnp-type) + (declare (type string cpp-type capnp-type)) + (push (cons cpp-type capnp-type) *capnp-type-converters*)) + +(defun capnp-type<-cpp-type (cpp-type) + (typecase cpp-type + (cpp-primitive-type + (when (member cpp-type '(:int :uint)) + (error "Unable to get Capnp type for integer without specified width.")) + ;; Delete the _t suffix + (cl-ppcre:regex-replace "_t$" (string-downcase cpp-type :start 1) "")) + (string + (let ((type-decl (parse-cpp-type-declaration cpp-type))) + (cond + ((string= "shared_ptr" (cpp-type-decl-name type-decl)) + (let ((class (find-cpp-class + ;; TODO: Use full type + (cpp-type-decl-name (first (cpp-type-decl-type-args type-decl)))))) + (unless class + (error "Unable to determine base type for '~A'; use :capnp-type" + cpp-type)) + (let* ((parents (capnp-union-parents-rec class)) + (top-parent (if parents (car (last parents)) (cpp-class-name class)))) + (format nil "Utils.SharedPtr(~A)" (cpp-type-name top-parent))))) + ((string= "vector" (cpp-type-decl-name type-decl)) + (format nil "List(~A)" + (capnp-type<-cpp-type + (string<-cpp-type-decl (first (cpp-type-decl-type-args type-decl)))))) + ((string= "optional" (cpp-type-decl-name type-decl)) + (format nil "Utils.Optional(~A)" + (capnp-type<-cpp-type + (string<-cpp-type-decl (first (cpp-type-decl-type-args type-decl)))))) + ((assoc cpp-type *capnp-type-converters* :test #'string=) + (cdr (assoc cpp-type *capnp-type-converters* :test #'string=))) + (t (cpp-type-name cpp-type))))) + ;; Capnp only accepts uppercase first letter in types (PascalCase), so + ;; this is the same as our conversion to C++ type name. + (otherwise (cpp-type-name cpp-type)))) + +(defun capnp-schema-for-enum (cpp-enum) + "Generate Cap'n Proto serialization schema for CPP-ENUM" + (declare (type cpp-enum cpp-enum)) + (with-output-to-string (s) + (format s "enum ~A {~%" (cpp-type-name (cpp-enum-symbol cpp-enum))) + (loop for val in (cpp-enum-values cpp-enum) and field-number from 0 + do (format s " ~A @~A;~%" + (string-downcase (cpp-type-name val) :end 1) + field-number)) + (write-line "}" s))) + +(defun capnp-schema (cpp-class) + "Generate Cap'n Proto serialiation schema for CPP-CLASS" + (declare (type (or cpp-class cpp-enum symbol) cpp-class)) + (when (null cpp-class) + (return-from capnp-schema)) + (when (cpp-enum-p cpp-class) + (return-from capnp-schema (capnp-schema-for-enum cpp-class))) + (let ((class-name (if (symbolp cpp-class) cpp-class (cpp-class-name cpp-class))) + (members (when (cpp-class-p cpp-class) (cpp-class-members cpp-class))) + (inner-types (when (cpp-class-p cpp-class) (cpp-class-inner-types cpp-class))) + (union-subclasses (capnp-union-subclasses cpp-class)) + (type-params (when (cpp-class-p cpp-class) (cpp-class-type-params cpp-class))) + (capnp-type-args (when (cpp-class-p cpp-class) + (capnp-opts-type-args (cpp-class-capnp-opts cpp-class)))) + (field-number 0)) + (when (and type-params (not capnp-type-args)) + (error "Don't know how to create schema for template class '~A'" class-name)) + (when (and capnp-type-args union-subclasses) + (error "Don't know how to handle templates and inheritance of ~A" class-name)) + (flet ((field-name<-symbol (symbol) + "Get Capnp compatible field name (camelCase)." + (string-downcase (cpp-type-name symbol) :end 1))) + (multiple-value-bind (union-parents compose-parents) + (capnp-union-and-compose-parents cpp-class) + (when (> (list-length union-parents) 1) + (error "Class ~A has multiple inheritance. Use :inherit-compose for + remaining parents." class-name)) + (with-output-to-string (s) + (format s "struct ~A {~%" (cpp-type-name class-name)) + (dolist (compose compose-parents) + (format s " ~A @~A :~A;~%" + (field-name<-symbol compose) + field-number + (capnp-type<-cpp-type compose)) + (incf field-number)) + (when capnp-type-args + (write-line " union {" s) + (dolist (type-arg capnp-type-args) + (format s " ~A @~A :Void;~%" (field-name<-symbol type-arg) field-number) + (incf field-number)) + (write-line " }" s)) + (dolist (member members) + (format s " ~A @~A :~A;~%" + (field-name<-symbol (cpp-member-symbol member)) + field-number + (if (cpp-member-capnp-type member) + (cpp-member-capnp-type member) + (capnp-type<-cpp-type (cpp-member-type member)))) + (incf field-number)) + (dolist (inner inner-types) + (write-line (capnp-schema inner) s)) + (when union-subclasses + (write-line " union {" s) + (dolist (subclass union-subclasses) + (format s " ~A @~A :~A;~%" + (field-name<-symbol (cpp-class-name subclass)) + field-number + (capnp-type<-cpp-type (cpp-class-name subclass))) + (incf field-number)) + (write-line " }" s)) + (write-line "}" s)))))) + +;;; Capnp C++ serialization code generation +;;; +;;; Algorithm is closely tied with the generated schema (see above). +;;; +;;; 1) Generate the method declaration. +;;; +;;; Two problems arise: +;;; * inheritance and +;;; * helper arguments (for tracking pointers or similar). +;;; +;;; The method will always take a pointer to a capnp::::Builder +;;; class. Additional arguments are optional, and are supplied when declaring +;;; that the class should be serialized with capnp. +;;; +;;; To determine the concrete T we need to know whether this class is a +;;; derived one or is inherited from. If it is, then T needs to be the +;;; top-most parent that is modeled by union and not composition. (For the +;;; inheritance modeling problem, refer to the description of schema +;;; generation.) Obviously, the method now needs to be virtual. If this class +;;; has no inheritance in any direction, then we just use the class name for +;;; T prepended with capnp:: namespace. +;;; +;;; Helper arguments are obtained from SAVE-ARGS of `CAPNP-OPTS'. +;;; +;;; 2) Generate parent calls for serialization (if we have parent classes). +;;; +;;; For the first (and only) parent which is modeled through union, generate a +;;; ::Save call. The call is passed all of the arguments from out +;;; function declaration. +;;; +;;; Find our own concrete builder by traversing through the union schema of +;;; the base builder. It is expected (and required) that the parent call has +;;; initialized them correctly. We just need to initialize the most concrete +;;; builder. +;;; +;;; Other parents are required to be modelled through composition. Therefore, +;;; we generate calls to parents by passing builders for the composed structs. +;;; auto parent_builder = builder->initParent(); +;;; Parent::Save(&parent_builder); +;;; Any additional helper arguments are also passed to the above call. +;;; +;;; 3) Generate member serialization. +;;; +;;; For primitive typed member, generate builder->setMember(member_); calls. +;;; +;;; For std types, generate hardcoded calls to our wrapper functions. Most of +;;; these require a lambda function which serializes the element inside the +;;; std class. This can be done recursively with this step. +;;; +;;; For composite types, check whether we have been given a custom save +;;; invocation. If not, assume that the type has a member function called Save +;;; which expects a builder for that type and any additional helper arguments. + +(defun capnp-extra-args (cpp-class save-or-load) + "Get additional arguments to Save/Load function for CPP-CLASS." + (declare (type cpp-class cpp-class) + (type (member :save :load) save-or-load)) + (loop for parent in (cons (cpp-class-name cpp-class) (capnp-union-parents-rec cpp-class)) + for opts = (cpp-class-capnp-opts (find-cpp-class parent)) + for args = (ecase save-or-load + (:save (capnp-opts-save-args opts)) + (:load (capnp-opts-load-args opts))) + when args return args)) + +(defun capnp-save-declaration (cpp-class &key (inline t)) + "Generate Cap'n Proto save function declaration for CPP-CLASS. If +INLINE is NIL, the declaration is namespaced for the class so that it can be +used for outside definition." + (declare (type cpp-class cpp-class)) + (let* ((parents (capnp-union-parents-rec cpp-class)) + (top-parent-class (if parents + (cpp-class-full-name (find-cpp-class (car (last parents)))) + (cpp-class-full-name cpp-class))) + (builder-arg + (list (if parents 'base-builder 'builder) + (format nil "capnp::~A::Builder *" top-parent-class)))) + (cpp-method-declaration + cpp-class "Save" :args (cons builder-arg (capnp-extra-args cpp-class :save)) + :virtual (and (not parents) (capnp-union-subclasses cpp-class)) + :const t :override parents :inline inline))) + +(defun capnp-save-default (member-name member-type member-builder) + "Generate the default call to save for member." + (declare (type string member-name member-type member-builder)) + (let* ((type (parse-cpp-type-declaration member-type)) + (type-name (cpp-type-decl-name type))) + (when (member type-name '("unique_ptr" "shared_ptr" "vector") :test #'string=) + (error "Use a custom :capnp-save function for ~A ~A" type-name member-name)) + (let* ((cpp-class (find-cpp-class type-name)) ;; TODO: full type-name search + (extra-args (when cpp-class + (mapcar (lambda (name-and-type) + (cpp-variable-name (first name-and-type))) + (capnp-extra-args cpp-class :save))))) + (format nil "~A.Save(&~A~{, ~A~});" + member-name member-builder extra-args)))) + +(defun capnp-save-code (cpp-class) + "Generate Cap'n Proto saving code for CPP-CLASS" + (declare (type cpp-class cpp-class)) + (with-output-to-string (s) + (format s "~A {~%" (capnp-save-declaration cpp-class :inline nil)) + (flet ((parent-args (parent) + (mapcar (lambda (name-and-type) + (cpp-variable-name (first name-and-type))) + (capnp-extra-args (find-cpp-class parent) :save)))) + (multiple-value-bind (direct-union-parents compose-parents) + (capnp-union-and-compose-parents cpp-class) + (declare (ignore direct-union-parents)) + ;; Handle the union inheritance calls first. + (let ((parents (capnp-union-parents-rec cpp-class))) + (when parents + (let ((first-parent (first parents))) + (format s " ~A::Save(base_builder~{, ~A~});~%" + (cpp-type-name first-parent) (parent-args first-parent))) + (when (or compose-parents (cpp-class-members cpp-class)) + (format s " auto ~A_builder = base_builder->~{get~A().~}init~A();~%" + (cpp-variable-name (cpp-class-name cpp-class)) + (mapcar #'cpp-type-name (cdr (reverse parents))) + (cpp-type-name (cpp-class-name cpp-class))) + (format s " auto *builder = &~A_builder;~%" + (cpp-variable-name (cpp-class-name cpp-class)))))) + ;; Now handle composite inheritance calls. + (dolist (parent compose-parents) + (write-line "{" s) + (format s " auto ~A_builder = builder->init~A();~%" + (cpp-variable-name parent) (cpp-type-name parent)) + (format s " ~A::Save(&~A_builder~{, ~A~});~%" + (cpp-type-name parent) (cpp-variable-name parent) (parent-args parent)) + (write-line "}" s)))) + ;; Set the template instantiations + (when (and (capnp-opts-type-args (cpp-class-capnp-opts cpp-class)) + (/= 1 (list-length (cpp-class-type-params cpp-class)))) + (error "Don't know how to save templated class ~A" (cpp-class-name cpp-class))) + (let ((type-param (first (cpp-class-type-params cpp-class)))) + (dolist (type-arg (capnp-opts-type-args (cpp-class-capnp-opts cpp-class))) + (format s " if (std::is_same<~A, ~A>::value) { builder->set~A(); }" + (cpp-type-name type-arg) (cpp-type-name type-param) (cpp-type-name type-arg)))) + (dolist (member (cpp-class-members cpp-class)) + (let ((member-name (cpp-member-name member :struct (cpp-class-structp cpp-class))) + (member-builder (format nil "~A_builder" (cpp-member-name member :struct t))) + (capnp-name (cpp-type-name (cpp-member-symbol member)))) + (cond + ((cpp-primitive-type-p (cpp-member-type member)) + (format s " builder->set~A(~A);~%" capnp-name member-name)) + (t + (write-line "{" s) ;; Enclose larger save code in new scope + (let ((size (if (string= "vector" (cpp-type-decl-name + (parse-cpp-type-declaration + (cpp-member-type member)))) + (format nil "~A.size()" member-name) + ""))) + (if (cpp-member-capnp-init member) + (format s " auto ~A = builder->init~A(~A);~%" + member-builder capnp-name size) + (setf member-builder "builder"))) + (if (cpp-member-capnp-save member) + (format s " ~A~%" + (cpp-code (funcall (cpp-member-capnp-save member) + member-builder member-name))) + (write-line (capnp-save-default member-name + (cpp-member-type member) + member-builder) + s)) + (write-line "}" s))))) + (write-line "}" s))) + +;;; Capnp C++ deserialization code generation +;;; +;;; This is almost the same as serialization, but with a special case for +;;; handling serialized pointers to base classes. +;;; +;;; We need to generate a static Construct function, which dispatches on the +;;; union of the Capnp struct schema. This needs to recursively traverse +;;; unions to instantiate a concrete C++ class. +;;; +;;; When loading such a pointer, first we call Construct and then follow it +;;; with calling Load on the constructed instance. + +(defun capnp-construct-declaration (cpp-class &key (inline t)) + "Generate Cap'n Proto construct function declaration for CPP-CLASS. If +INLINE is NIL, the declaration is namespaced for the class so that it can be +used for outside definition." + (declare (type cpp-class cpp-class)) + ;; Don't generate Construct if this class is only inherited as composite. + (when (and (not (capnp-union-subclasses cpp-class)) + (direct-subclasses-of cpp-class)) + (return-from capnp-construct-declaration)) + (let ((reader-type (format nil "const capnp::~A::Reader &" + (cpp-class-full-name cpp-class :type-args nil)))) + (cpp-method-declaration + cpp-class "Construct" :args (list (list 'reader reader-type)) + :returns (format nil "std::unique_ptr<~A>" (cpp-class-full-name cpp-class)) + :inline inline :static t))) + +(defun capnp-construct-code (cpp-class) + "Generate Cap'n Proto construct code for CPP-CLASS" + (declare (type cpp-class cpp-class)) + (let ((construct-declaration (capnp-construct-declaration cpp-class :inline nil))) + (unless construct-declaration + (return-from capnp-construct-code)) + (let ((class-name (cpp-class-name cpp-class)) + (union-subclasses (capnp-union-subclasses cpp-class))) + (with-output-to-string (s) + (format s "~A {~%" construct-declaration) + (if (not union-subclasses) + ;; No inheritance, just instantiate the concrete class. + (format s " return std::unique_ptr<~A>(new ~A());~%" + (cpp-type-name class-name) (cpp-type-name class-name)) + ;; Inheritance, so forward the Construct. + (progn + (write-line " switch (reader.which()) {" s) + (dolist (subclass union-subclasses) + (format s " case capnp::~A::~A:~%" + (cpp-type-name class-name) + (cpp-constant-name (cpp-class-name subclass))) + (let ((subclass-name (cpp-type-name (cpp-class-name subclass)))) + (if (capnp-opts-type-args (cpp-class-capnp-opts subclass)) + ;; Handle template instantiation + (progn + (format s " switch (reader.get~A().which()) {~%" subclass-name) + (dolist (type-arg (capnp-opts-type-args (cpp-class-capnp-opts subclass))) + (format s " case capnp::~A::~A: return ~A<~A>::Construct(reader.get~A());~%" + subclass-name (cpp-constant-name type-arg) + subclass-name (cpp-type-name type-arg) + subclass-name)) + (write-line " }" s)) + ;; Just forward the construct normally. + (format s " return ~A::Construct(reader.get~A());~%" + subclass-name subclass-name)))) + (write-line " }" s))) ;; close switch + (write-line "}" s))))) ;; close function + +(defun capnp-load-default (member-name member-type member-reader) + "Generate default load call for member." + (declare (type string member-name member-type member-reader)) + (let* ((type (parse-cpp-type-declaration member-type)) + (type-name (cpp-type-decl-name type))) + (when (member type-name '("unique_ptr" "shared_ptr" "vector") :test #'string=) + (error "Use a custom :capnp-load function for ~A ~A" type-name member-name)) + (let* ((cpp-class (find-cpp-class type-name)) ;; TODO: full type-name search + (extra-args (when cpp-class + (mapcar (lambda (name-and-type) + (cpp-variable-name (first name-and-type))) + (capnp-extra-args cpp-class :load))))) + (format nil "~A.Load(~A~{, ~A~});" + member-name member-reader extra-args)))) + +(defun capnp-load-declaration (cpp-class &key (inline t)) + "Generate Cap'n Proto load function declaration for CPP-CLASS. If +INLINE is NIL, the declaration is namespaced for the class so that it can be +used for outside definition." + (declare (type cpp-class cpp-class)) + (let* ((parents (capnp-union-parents-rec cpp-class)) + (top-parent-class (if parents + (cpp-class-full-name (find-cpp-class (car (last parents)))) + (cpp-class-full-name cpp-class))) + (reader-arg + (list (if parents 'base-reader 'reader) + (format nil "const capnp::~A::Reader &" top-parent-class)))) + (cpp-method-declaration + cpp-class "Load" :args (cons reader-arg (capnp-extra-args cpp-class :load)) + :virtual (and (not parents) (capnp-union-subclasses cpp-class)) + :override parents :inline inline))) + +(defun capnp-load-code (cpp-class) + "Generate Cap'n Proto loading code for CPP-CLASS" + (declare (type cpp-class cpp-class)) + (with-output-to-string (s) + (format s "~A {~%" (capnp-load-declaration cpp-class :inline nil)) + (flet ((parent-args (parent) + (mapcar (lambda (name-and-type) + (cpp-variable-name (first name-and-type))) + (capnp-extra-args (find-cpp-class parent) :load)))) + (multiple-value-bind (direct-union-parents compose-parents) + (capnp-union-and-compose-parents cpp-class) + (declare (ignore direct-union-parents)) + ;; Handle the union inheritance calls first. + (let ((parents (capnp-union-parents-rec cpp-class))) + (when parents + (let ((first-parent (first parents))) + (format s " ~A::Load(base_reader~{, ~A~});~%" + (cpp-type-name first-parent) (parent-args first-parent))) + (when (or compose-parents (cpp-class-members cpp-class)) + (format s " auto reader = base_reader.~{get~A().~}get~A();~%" + (mapcar #'cpp-type-name (cdr (reverse parents))) + (cpp-type-name (cpp-class-name cpp-class)))) + ;; Now handle composite inheritance calls. + (dolist (parent compose-parents) + (write-line "{" s) + (format s " auto ~A_reader = reader.get~A();~%" + (cpp-variable-name parent) (cpp-type-name parent)) + (format s " ~A::Load(~A_reader~{, ~A~});~%" + (cpp-type-name parent) (cpp-variable-name parent) (parent-args parent)) + (write-line "}" s)))))) + (dolist (member (cpp-class-members cpp-class)) + (let ((member-name (cpp-member-name member :struct (cpp-class-structp cpp-class))) + (member-reader (format nil "~A_reader" (cpp-member-name member :struct t))) + (capnp-name (cpp-type-name (cpp-member-symbol member)))) + (cond + ((cpp-primitive-type-p (cpp-member-type member)) + (format s " ~A = reader.get~A();~%" member-name capnp-name)) + (t + (write-line "{" s) ;; Enclose larger load code in new scope + (if (cpp-member-capnp-init member) + (format s " auto ~A = reader.get~A();~%" member-reader capnp-name) + (setf member-reader "reader")) + (if (cpp-member-capnp-load member) + (format s " ~A~%" + (cpp-code (funcall (cpp-member-capnp-load member) + member-reader member-name))) + (write-line (capnp-load-default member-name + (cpp-member-type member) + member-reader) s)) + (write-line "}" s))))) + (write-line "}" s))) + +(defvar *capnp-imports* nil + "List of pairs (namespace, import-file), which will be imported in Cap'n + Proto schema with the syntax 'using Namespace = import import-file'") + +(defun capnp-import (namespace import-file) + "Import the IMPORT-FILE to Cap'n Proto aliased using NAMESPACE." + (declare (type symbol namespace) + (type string import-file)) + (push (cons namespace import-file) *capnp-imports*)) + +(defvar *capnp-namespace* nil + "Name of the namespace where Cap'n Proto generated C++ will be.") + +(defun capnp-namespace (namespace) + "Set the Cap'n Proto generated c++ namespace." + (declare (type string namespace)) + (setf *capnp-namespace* namespace)) + +(defvar *cpp-namespaces* nil + "Stack of C++ namespaces we are generating the code in.") + +(defmacro namespace (name) + "Push the NAME to currently set namespaces." + (declare (type symbol name)) + (let ((cpp-namespace (cpp-variable-name name))) + `(progn + (push ,cpp-namespace *cpp-namespaces*) + (make-raw-cpp + :string ,(format nil "~%namespace ~A {~%" cpp-namespace))))) + +(defun pop-namespace () + (pop *cpp-namespaces*) + #>cpp } cpp<#) + +(defvar *cpp-inner-types* nil + "List of cpp types defined inside an enclosing class or struct") + +(defvar *cpp-enclosing-class* nil + "Symbol name of the `CPP-CLASS' inside which inner types are defined.") + +(defmacro define-enum (name maybe-documentation &rest values) + "Define a C++ enum. Documentation is optional. Syntax is: + +;; (define-enum name +;; [documentation-string] +;; value1 value2 ...)" + (declare (type symbol name) + (type (or string symbol) maybe-documentation)) + (let ((documentation (when (stringp maybe-documentation) maybe-documentation)) + (all-values (if (symbolp maybe-documentation) + (cons maybe-documentation values) + values)) + (enum (gensym (format nil "ENUM-~A" name)))) + `(let ((,enum (make-cpp-enum :symbol ',name + :documentation ,documentation + :values ',all-values + :enclosing-class *cpp-enclosing-class*))) + (prog1 ,enum + (push ,enum *cpp-inner-types*))))) (defmacro define-class (name super-classes slots &rest options) "Define a C++ class. Syntax is: -(define-class name (list-of-super-classes) - ((c++-slot-definition)*) - (:class-option option-value)*) +;; (define-class name (list-of-super-classes) +;; ((c++-slot-definition)*) +;; (:class-option option-value)*) Class name may be a list where the first element is the class name, while others are template arguments. For example: -(define-class (optional t-value) - ...) +;; (define-class (optional t-value) +;; ...) defines a templated C++ class: @@ -328,7 +1084,7 @@ template class Optional { ... }; Each C++ member/slot definition is of the form: - (name cpp-type slot-options) +;; (name cpp-type slot-options) slot-options are keyword arguments. Currently supported options are: * :initval -- initializer value for the member, a C++ string or a number. @@ -343,64 +1099,195 @@ Currently supported class-options are: * :public -- additional C++ code in public scope. * :protected -- additional C++ code in protected scope. * :private -- additional C++ code in private scope. - * :serialize -- only :boost is a valid value, setting this will generate - boost serialization code for the class members. + * :serialize -- only :boost and :capnp are valid values. Setting :boost + will generate boost serialization code for the class members. Setting + :capnp will generate the Cap'n Proto serialization code for the class + members. You may specifiy additional options after :capnp to fill the + `CAPNP-OPTS' slots. Larger example: -(lcp:define-class derived (base) - ((val :int :reader t :initval 42)) - (:public #>cpp void set_val(int new_val) { val_ = new_val; } cpp<#) - (:serialize :boost)) +;; (lcp:define-class derived (base) +;; ((val :int :reader t :initval 42)) +;; (:public #>cpp void set_val(int new_val) { val_ = new_val; } cpp<#) +;; (:serialize :boost)) Generates C++: -class Derived : public Base { - public: - void set_val(int new_val) { val_ = new_val; } - auto val() { return val_; } // autogenerated from :reader t - - private: - friend class boost::serialization::access; - template - void serialize(TArchive &ar, unsigned int) { - ar & boost::serialization::base_object(*this); - ar & val_; - } - - int val_ = 42; // :initval is assigned -};" +;; class Derived : public Base { +;; public: +;; void set_val(int new_val) { val_ = new_val; } +;; auto val() { return val_; } // autogenerated from :reader t +;; +;; private: +;; friend class boost::serialization::access; +;; template +;; void serialize(TArchive &ar, unsigned int) { +;; ar & boost::serialization::base_object(*this); +;; ar & val_; +;; } +;; +;; int val_ = 42; // :initval is assigned +;; };" (let ((structp (second (assoc :structp options)))) - (flet ((parse-slot (slot-name type &key initval reader scope - documentation save-fun load-fun) - (let ((scope (if scope - scope - (if structp :public :private)))) + (flet ((parse-slot (slot-name type &rest kwargs + &key reader scope &allow-other-keys) + (let ((scope (if scope scope (if structp :public :private)))) (when (and structp reader (eq :private scope)) (error "Slot ~A is declared private with reader in a struct. You should use define-class" slot-name)) (when (and structp reader (eq :public scope)) (error "Slot ~A is public, you shouldn't specify :reader" slot-name)) - `(make-cpp-member :symbol ',slot-name :type ,type :initval ,initval - :reader ,reader :scope ,scope - :documentation ,documentation - :save-fun ,save-fun :load-fun ,load-fun)))) + `(make-cpp-member :symbol ',slot-name :type ,type :scope ,scope + ,@kwargs)))) (let ((members (mapcar (lambda (s) (apply #'parse-slot s)) slots)) (class-name (if (consp name) (car name) name)) (type-params (when (consp name) (cdr name))) - (class (gensym (format nil "CLASS-~A" name)))) + (class (gensym (format nil "CLASS-~A" name))) + (serialize (cdr (assoc :serialize options)))) `(let ((,class - (make-cpp-class :name ',class-name :super-classes ',super-classes - :type-params ',type-params - :structp ,(second (assoc :structp options)) - :members (list ,@members) - :documentation ,(second (assoc :documentation options)) - :public (list ,@(cdr (assoc :public options))) - :protected (list ,@(cdr (assoc :protected options))) - :private (list ,@(cdr (assoc :private options)))))) + (let ((*cpp-inner-types* nil) + (*cpp-enclosing-class* ',class-name)) + (make-cpp-class :name ',class-name :super-classes ',super-classes + :type-params ',type-params + :structp ,(second (assoc :structp options)) + :members (list ,@members) + :documentation ,(second (assoc :documentation options)) + :public (list ,@(cdr (assoc :public options))) + :protected (list ,@(cdr (assoc :protected options))) + :private (list ,@(cdr (assoc :private options))) + :capnp-opts ,(when (member :capnp serialize) + `(make-capnp-opts ,@(cdr (member :capnp serialize)))) + :namespace (reverse *cpp-namespaces*) + ;; Set inner types at the end. This works + ;; because CL standard specifies order of + ;; evaluation from left to right. + :inner-types *cpp-inner-types*)))) (prog1 ,class - ,(when (eq :boost (cadr (assoc :serialize options))) + (push ,class *cpp-classes*) + ;; Set the parent's inner types + (push ,class *cpp-inner-types*) + (setf (cpp-class-enclosing-class ,class) *cpp-enclosing-class*) + ,(when (eq :boost (car serialize)) `(setf (cpp-class-private ,class) (append (cpp-class-private ,class) (boost-serialization ,class)))))))))) (defmacro define-struct (name super-classes slots &rest options) `(define-class ,name ,super-classes ,slots (:structp t) ,@options)) + +(defun read-lcp (filepath) + "Read the FILEPATH and return a list of C++ meta information that should be +formatted and output." + (with-open-file (in-stream filepath) + (let ((stream-pos 0)) + (handler-case + (loop for form = (read-preserving-whitespace in-stream nil 'eof) + until (eq form 'eof) + for res = (handler-case (eval form) + (error (err) + (file-position in-stream 0) ;; start of stream + (error "~%~A:~A: error:~%~%~A~%~%in:~%~%~A" + (uiop:native-namestring filepath) + (count-newlines in-stream :stop-position (1+ stream-pos)) + err form))) + do (setf stream-pos (file-position in-stream)) + when (typep res '(or raw-cpp cpp-class cpp-enum)) + collect res) + (end-of-file () + (file-position in-stream 0) ;; start of stream + (error "~%~A:~A:error: READ error, did you forget a closing ')'?" + (uiop:native-namestring filepath) + (count-newlines in-stream + :stop-position (1+ stream-pos)))))))) + +(defun generate-capnp (cpp-classes &key capnp-file capnp-id cpp-file hpp-file lcp-file) + "Generate Cap'n Proto serialization code for given CPP-CLASSES. The schema +is written to CAPNP-FILE using the CAPNP-ID. The C++ serialization code is +written to CPP-FILE. This source file will include the provided HPP-FILE. +Original LCP-FILE is used just to insert a comment about the source of the +code generation." + (with-open-file (out capnp-file :direction :output :if-exists :supersede) + (format out "# Autogenerated using LCP from '~A'~%~%" lcp-file) + (format out "~A;~%~%" capnp-id) + (write-line "using Cxx = import \"/capnp/c++.capnp\";" out) + (format out "$Cxx.namespace(\"~A::capnp\");~%~%" *capnp-namespace*) + (dolist (capnp-import *capnp-imports* (terpri out)) + (format out "using ~A = import ~S;~%" + (remove #\- (string-capitalize (car capnp-import))) + (cdr capnp-import))) + (dolist (cpp-class cpp-classes) + ;; Generate schema only for top level classes, inner classes are handled + ;; inside the generation of the enclosing class. + (unless (cpp-class-enclosing-class cpp-class) + (let ((schema (capnp-schema cpp-class))) + (when schema (write-line schema out)))))) + ;; Now generate the save/load C++ code in the cpp file. + (with-open-file (out cpp-file :direction :output :if-exists :supersede) + (format out "// Autogenerated using LCP from '~A'~%~%" lcp-file) + (write-line "#include \"utils/serialization.hpp\"" out) + (format out "#include \"~A\"~%~%" hpp-file) + (let (open-namespaces) + (dolist (cpp-class cpp-classes) + ;; Check if we need to open or close namespaces + (loop for namespace in (cpp-class-namespace cpp-class) + with unmatched = open-namespaces do + (if (string= namespace (car unmatched)) + (setf unmatched (cdr unmatched)) + (progn + (dolist (to-close unmatched) + (declare (ignore to-close)) + (write-line "}" out)) + (format out "namespace ~A {~%~%" namespace)))) + (setf open-namespaces (cpp-class-namespace cpp-class)) + ;; Output the serialization code + (format out "// Serialize code for ~A~%~%" + (cpp-type-name (cpp-class-name cpp-class))) + (let ((save-code (capnp-save-code cpp-class)) + (construct-code (capnp-construct-code cpp-class)) + (load-code (capnp-load-code cpp-class))) + (when save-code (write-line save-code out)) + (when construct-code (write-line construct-code out)) + (when load-code (write-line load-code out)))) + ;; Close remaining namespaces + (dolist (to-close open-namespaces) + (declare (ignore to-close)) + (write-line "}" out))))) + +(defun process-file (lcp-file &key capnp-id) + "Process a LCP-FILE and write the output to .hpp file in the same directory. +If CAPNP-ID is passed, generates the Cap'n Proto schema to .capnp file in the +same directory, while the loading code is generated in LCP-FILE.cpp source +file." + (multiple-value-bind (filename extension) + (uiop:split-name-type lcp-file) + (assert (string= (string-downcase extension) "lcp")) + (let ((hpp-file (concatenate 'string filename ".hpp")) + ;; Unlike hpp, for cpp file use the full path. This allows us to + ;; have our own accompanying .cpp files + (cpp-file (concatenate 'string lcp-file ".cpp")) + (capnp-file (concatenate 'string filename ".capnp")) + ;; Reset globals + (*capnp-namespace* nil) + (*capnp-imports* nil) + (*capnp-type-converters* nil) + (*cpp-inner-types* nil) + ;; Don't reset *cpp-classes* if we want to have support for + ;; procesing multiple files. + ;; (*cpp-classes* nil) + ) + ;; First read and evaluate the whole file, then output the evaluated + ;; cpp-code. This allows us to generate code which may rely on + ;; evaluation done after the code definition. + (with-open-file (out hpp-file :direction :output :if-exists :supersede) + (format out "// Autogenerated using LCP from '~A'~%~%" lcp-file) + (dolist (res (read-lcp lcp-file)) + (write-line (cpp-code res) out))) + (when *cpp-namespaces* + (error "Unclosed namespaces: ~A" (reverse *cpp-namespaces*))) + ;; If we have a capnp-id, generate the schema + (when capnp-id + (let ((classes-for-capnp + (remove-if (complement #'cpp-class-capnp-opts) *cpp-classes*))) + (when classes-for-capnp + (generate-capnp classes-for-capnp :capnp-file capnp-file :capnp-id capnp-id + :cpp-file cpp-file :hpp-file (file-namestring hpp-file) + :lcp-file lcp-file))))))) diff --git a/src/query/common.capnp b/src/query/common.capnp new file mode 100644 index 000000000..5dd38e778 --- /dev/null +++ b/src/query/common.capnp @@ -0,0 +1,15 @@ +@0xcbc2c66202fdf643; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("query::capnp"); + +using Ast = import "/query/frontend/ast/ast.capnp"; + +enum GraphView { + old @0; + new @1; +} + +struct TypedValueVectorCompare { + ordering @0 :List(Ast.Ordering); +} diff --git a/src/query/common.cpp b/src/query/common.cpp index 7ffb01de0..6261d989d 100644 --- a/src/query/common.cpp +++ b/src/query/common.cpp @@ -8,6 +8,7 @@ #include "glog/logging.h" #include "query/exceptions.hpp" +#include "utils/serialization.hpp" #include "utils/string.hpp" namespace query { @@ -280,6 +281,28 @@ bool TypedValueVectorCompare::TypedValueCompare(const TypedValue &a, } } +void TypedValueVectorCompare::Save( + capnp::TypedValueVectorCompare::Builder *builder) const { + auto ordering_builder = builder->initOrdering(ordering_.size()); + for (size_t i = 0; i < ordering_.size(); ++i) { + ordering_builder.set(i, ordering_[i] == Ordering::ASC + ? capnp::Ordering::ASC + : capnp::Ordering::DESC); + } +} + +void TypedValueVectorCompare::Load( + const capnp::TypedValueVectorCompare::Reader &reader) { + std::vector ordering; + ordering.reserve(reader.getOrdering().size()); + for (auto ordering_reader : reader.getOrdering()) { + ordering.push_back(ordering_reader == capnp::Ordering::ASC + ? Ordering::ASC + : Ordering::DESC); + } + ordering_ = ordering; +} + template void SwitchAccessor(TAccessor &accessor, GraphView graph_view) { switch (graph_view) { diff --git a/src/query/common.hpp b/src/query/common.hpp index 91bb8289a..8b4ff26cc 100644 --- a/src/query/common.hpp +++ b/src/query/common.hpp @@ -7,6 +7,8 @@ #include "query/frontend/ast/ast.hpp" #include "query/typed_value.hpp" +#include "query/common.capnp.h" + namespace query { // These are the functions for parsing literals and parameter names from @@ -37,7 +39,7 @@ void ReconstructTypedValue(TypedValue &value); // Does lexicographical ordering of elements based on the above // defined TypedValueCompare, and also accepts a vector of Orderings // the define how respective elements compare. -class TypedValueVectorCompare { +class TypedValueVectorCompare final { public: TypedValueVectorCompare() {} explicit TypedValueVectorCompare(const std::vector &ordering) @@ -47,6 +49,9 @@ class TypedValueVectorCompare { const auto &ordering() const { return ordering_; } + void Save(capnp::TypedValueVectorCompare::Builder *builder) const; + void Load(const capnp::TypedValueVectorCompare::Reader &reader); + private: std::vector ordering_; diff --git a/src/query/frontend/ast/ast.cpp b/src/query/frontend/ast/ast.cpp index 1d66032c5..154f59007 100644 --- a/src/query/frontend/ast/ast.cpp +++ b/src/query/frontend/ast/ast.cpp @@ -309,7 +309,7 @@ void MapLiteral::Save(capnp::BaseLiteral::Builder *base_literal_builder, auto key_builder = entry_builder.getKey(); key_builder.setFirst(entry.first.first); auto storage_property_builder = key_builder.getSecond(); - entry.first.second.Save(storage_property_builder); + entry.first.second.Save(&storage_property_builder); auto value_builder = entry_builder.getValue(); if (entry.second) entry.second->Save(&value_builder, saved_uids); ++i; @@ -976,11 +976,10 @@ void LabelsTest::Save(capnp::LabelsTest::Builder *builder, auto expr_builder = builder->initExpression(); expression_->Save(&expr_builder, saved_uids); } - ::capnp::List::Builder common_builders = - builder->initLabels(labels_.size()); + auto common_builders = builder->initLabels(labels_.size()); for (size_t i = 0; i < labels_.size(); ++i) { auto common_builder = common_builders[i]; - labels_[i].Save(common_builder); + labels_[i].Save(&common_builder); } } @@ -1040,7 +1039,7 @@ void PropertyLookup::Save(capnp::PropertyLookup::Builder *builder, } builder->setPropertyName(property_name_); auto storage_property_builder = builder->initProperty(); - property_.Save(storage_property_builder); + property_.Save(&storage_property_builder); } void PropertyLookup::Load(const capnp::Tree::Reader &base_reader, @@ -1297,9 +1296,9 @@ void CreateIndex::Save(capnp::Clause::Builder *builder, void CreateIndex::Save(capnp::CreateIndex::Builder *builder, std::vector *saved_uids) { auto label_builder = builder->getLabel(); - label_.Save(label_builder); + label_.Save(&label_builder); auto property_builder = builder->getProperty(); - property_.Save(property_builder); + property_.Save(&property_builder); } CreateIndex *CreateIndex::Construct(const capnp::CreateIndex::Reader &reader, @@ -1458,13 +1457,13 @@ void RemoveLabels::Save(capnp::RemoveLabels::Builder *builder, auto id_builder = builder->getIdentifier(); identifier_->Save(&id_builder, saved_uids); } - ::capnp::List::Builder common_builders = - builder->initLabels(labels_.size()); + auto common_builders = builder->initLabels(labels_.size()); for (size_t i = 0; i < labels_.size(); ++i) { auto common_builder = common_builders[i]; - labels_[i].Save(common_builder); + labels_[i].Save(&common_builder); } } + void RemoveLabels::Load(const capnp::Tree::Reader &base_reader, AstTreeStorage *storage, std::vector *loaded_uids) { @@ -1628,11 +1627,10 @@ void SetLabels::Save(capnp::SetLabels::Builder *builder, auto id_builder = builder->getIdentifier(); identifier_->Save(&id_builder, saved_uids); } - ::capnp::List::Builder common_builders = - builder->initLabels(labels_.size()); + auto common_builders = builder->initLabels(labels_.size()); for (size_t i = 0; i < labels_.size(); ++i) { auto common_builder = common_builders[i]; - labels_[i].Save(common_builder); + labels_[i].Save(&common_builder); } } @@ -1827,11 +1825,10 @@ void CypherUnion::Save(capnp::CypherUnion::Builder *builder, single_query_->Save(&sq_builder, saved_uids); } builder->setDistinct(distinct_); - ::capnp::List::Builder symbol_builders = - builder->initUnionSymbols(union_symbols_.size()); + auto symbol_builders = builder->initUnionSymbols(union_symbols_.size()); for (size_t i = 0; i < union_symbols_.size(); ++i) { auto symbol_builder = symbol_builders[i]; - union_symbols_[i].Save(symbol_builder); + union_symbols_[i].Save(&symbol_builder); } } @@ -2007,16 +2004,15 @@ void NodeAtom::Save(capnp::NodeAtom::Builder *builder, auto key_builder = entry_builder.getKey(); key_builder.setFirst(entry.first.first); auto storage_property_builder = key_builder.getSecond(); - entry.first.second.Save(storage_property_builder); + entry.first.second.Save(&storage_property_builder); auto value_builder = entry_builder.getValue(); if (entry.second) entry.second->Save(&value_builder, saved_uids); ++i; } - ::capnp::List::Builder common_builders = - builder->initLabels(labels_.size()); + auto common_builders = builder->initLabels(labels_.size()); for (size_t i = 0; i < labels_.size(); ++i) { auto common_builder = common_builders[i]; - labels_[i].Save(common_builder); + labels_[i].Save(&common_builder); } } @@ -2102,11 +2098,10 @@ void EdgeAtom::Save(capnp::EdgeAtom::Builder *builder, break; } - ::capnp::List::Builder common_builders = - builder->initEdgeTypes(edge_types_.size()); + auto common_builders = builder->initEdgeTypes(edge_types_.size()); for (size_t i = 0; i < edge_types_.size(); ++i) { auto common_builder = common_builders[i]; - edge_types_[i].Save(common_builder); + edge_types_[i].Save(&common_builder); } ::capnp::List::Builder map_builder = @@ -2117,7 +2112,7 @@ void EdgeAtom::Save(capnp::EdgeAtom::Builder *builder, auto key_builder = entry_builder.getKey(); key_builder.setFirst(entry.first.first); auto storage_property_builder = key_builder.getSecond(); - entry.first.second.Save(storage_property_builder); + entry.first.second.Save(&storage_property_builder); auto value_builder = entry_builder.getValue(); if (entry.second) entry.second->Save(&value_builder, saved_uids); ++i; diff --git a/src/query/frontend/semantic/symbol.hpp b/src/query/frontend/semantic/symbol.hpp index 4620e46a6..f9e892fb4 100644 --- a/src/query/frontend/semantic/symbol.hpp +++ b/src/query/frontend/semantic/symbol.hpp @@ -42,29 +42,29 @@ class Symbol { bool user_declared() const { return user_declared_; } int token_position() const { return token_position_; } - void Save(capnp::Symbol::Builder &builder) { - builder.setName(name_); - builder.setPosition(position_); - builder.setUserDeclared(user_declared_); - builder.setTokenPosition(token_position_); + void Save(capnp::Symbol::Builder *builder) const { + builder->setName(name_); + builder->setPosition(position_); + builder->setUserDeclared(user_declared_); + builder->setTokenPosition(token_position_); switch (type_) { case Type::Any: - builder.setType(capnp::Symbol::Type::ANY); + builder->setType(capnp::Symbol::Type::ANY); break; case Type::Edge: - builder.setType(capnp::Symbol::Type::EDGE); + builder->setType(capnp::Symbol::Type::EDGE); break; case Type::EdgeList: - builder.setType(capnp::Symbol::Type::EDGE_LIST); + builder->setType(capnp::Symbol::Type::EDGE_LIST); break; case Type::Number: - builder.setType(capnp::Symbol::Type::NUMBER); + builder->setType(capnp::Symbol::Type::NUMBER); break; case Type::Path: - builder.setType(capnp::Symbol::Type::PATH); + builder->setType(capnp::Symbol::Type::PATH); break; case Type::Vertex: - builder.setType(capnp::Symbol::Type::VERTEX); + builder->setType(capnp::Symbol::Type::VERTEX); break; } } diff --git a/src/query/plan/operator.lcp b/src/query/plan/operator.lcp index 72c70906b..d6b2f9279 100644 --- a/src/query/plan/operator.lcp +++ b/src/query/plan/operator.lcp @@ -28,19 +28,25 @@ #include "utils/hashing/fnv.hpp" #include "utils/visitor.hpp" +#include "query/plan/operator.capnp.h" + namespace database { class GraphDbAccessor; } +cpp<# -namespace query { +(lcp:namespace query) +#>cpp class Context; class ExpressionEvaluator; class Frame; class SymbolTable; +cpp<# -namespace plan { +(lcp:namespace plan) +#>cpp /** @brief Base class for iteration cursors of @c LogicalOperator classes. * * Each @c LogicalOperator must produce a concrete @c Cursor, which provides @@ -129,82 +135,115 @@ class HierarchicalLogicalOperatorVisitor using LogicalOperatorLeafVisitor::Visit; using typename LogicalOperatorLeafVisitor::ReturnType; }; +cpp<# -/** Base class for logical operators. - * - * Each operator describes an operation, which is to be performed on the - * database. Operators are iterated over using a @c Cursor. Various operators - * can serve as inputs to others and thus a sequence of operations is formed. - */ -class LogicalOperator - : public ::utils::Visitable { - public: - virtual ~LogicalOperator() {} +(lcp:capnp-namespace "query::plan") - /** Constructs a @c Cursor which is used to run this operator. - * - * @param database::GraphDbAccessor Used to perform operations on the - * database. - */ - virtual std::unique_ptr MakeCursor( - database::GraphDbAccessor &) const = 0; +(lcp:capnp-import 'utils "/utils/serialization.capnp") +(lcp:capnp-import 'storage "/storage/types.capnp") +(lcp:capnp-import 'ast "/query/frontend/ast/ast.capnp") +(lcp:capnp-import 'semantic "/query/frontend/semantic/symbol.capnp") +(lcp:capnp-import 'common "/query/common.capnp") - /** Return @c Symbol vector where the query results will be stored. - * - * Currently, outputs symbols are generated in @c Produce and @c Union - * operators. @c Skip, @c Limit, @c OrderBy and @c Distinct propagate the - * symbols from @c Produce (if it exists as input operator). - * - * @param SymbolTable used to find symbols for expressions. - * @return std::vector used for results. - */ - virtual std::vector OutputSymbols(const SymbolTable &) const { - return std::vector(); - } +(lcp:capnp-type-conversion "Symbol" "Semantic.Symbol") +(lcp:capnp-type-conversion "storage::Label" "Storage.Common") +(lcp:capnp-type-conversion "storage::Property" "Storage.Common") +(lcp:capnp-type-conversion "storage::EdgeType" "Storage.Common") +(lcp:capnp-type-conversion "GraphView" "Common.GraphView") - /** - * Symbol vector whose values are modified by this operator sub-tree. - * - * This is different than @c OutputSymbols, because it returns all of the - * modified symbols, including those that may not be returned as the - * result of the query. Note that the modified symbols will not contain - * those that should not be read after the operator is processed. - * - * For example, `MATCH (n)-[e]-(m) RETURN n AS l` will generate `ScanAll (n) > - * Expand (e, m) > Produce (l)`. The modified symbols on Produce sub-tree will - * be `l`, the same as output symbols, because it isn't valid to read `n`, `e` - * nor `m` after Produce. On the other hand, modified symbols from Expand - * contain `e` and `m`, as well as `n`, while output symbols are empty. - * Modified symbols from ScanAll contain only `n`, while output symbols are - * also empty. - */ - virtual std::vector ModifiedSymbols(const SymbolTable &) const = 0; +(lcp:define-class logical-operator ("::utils::Visitable") + () + (:documentation + "Base class for logical operators. - /** - * Returns true if the operator takes only one input operator. - * NOTE: When this method returns true, you may use `input` and `set_input` - * methods. - */ - virtual bool HasSingleInput() const = 0; +Each operator describes an operation, which is to be performed on the +database. Operators are iterated over using a @c Cursor. Various operators +can serve as inputs to others and thus a sequence of operations is formed.") + (:public + #>cpp + virtual ~LogicalOperator() {} - /** - * Returns the input operator if it has any. - * NOTE: This should only be called if `HasSingleInput() == true`. - */ - virtual std::shared_ptr input() const = 0; - /** - * Set a different input on this operator. - * NOTE: This should only be called if `HasSingleInput() == true`. - */ - virtual void set_input(std::shared_ptr) = 0; + /** Constructs a @c Cursor which is used to run this operator. + * + * @param database::GraphDbAccessor Used to perform operations on the + * database. + */ + virtual std::unique_ptr MakeCursor( + database::GraphDbAccessor &) const = 0; - private: - friend class boost::serialization::access; + /** Return @c Symbol vector where the query results will be stored. + * + * Currently, outputs symbols are generated in @c Produce and @c Union + * operators. @c Skip, @c Limit, @c OrderBy and @c Distinct propagate the + * symbols from @c Produce (if it exists as input operator). + * + * @param SymbolTable used to find symbols for expressions. + * @return std::vector used for results. + */ + virtual std::vector OutputSymbols(const SymbolTable &) const { + return std::vector(); + } - template - void serialize(TArchive &, const unsigned int) {} -}; + /** + * Symbol vector whose values are modified by this operator sub-tree. + * + * This is different than @c OutputSymbols, because it returns all of the + * modified symbols, including those that may not be returned as the + * result of the query. Note that the modified symbols will not contain + * those that should not be read after the operator is processed. + * + * For example, `MATCH (n)-[e]-(m) RETURN n AS l` will generate `ScanAll (n) > + * Expand (e, m) > Produce (l)`. The modified symbols on Produce sub-tree will + * be `l`, the same as output symbols, because it isn't valid to read `n`, `e` + * nor `m` after Produce. On the other hand, modified symbols from Expand + * contain `e` and `m`, as well as `n`, while output symbols are empty. + * Modified symbols from ScanAll contain only `n`, while output symbols are + * also empty. + */ + virtual std::vector ModifiedSymbols(const SymbolTable &) const = 0; + /** + * Returns true if the operator takes only one input operator. + * NOTE: When this method returns true, you may use `input` and `set_input` + * methods. + */ + virtual bool HasSingleInput() const = 0; + + /** + * Returns the input operator if it has any. + * NOTE: This should only be called if `HasSingleInput() == true`. + */ + virtual std::shared_ptr input() const = 0; + /** + * Set a different input on this operator. + * NOTE: This should only be called if `HasSingleInput() == true`. + */ + virtual void set_input(std::shared_ptr) = 0; + + struct SaveHelper { + std::vector saved_ast_uids; + std::vector saved_ops; + }; + + struct LoadHelper { + AstTreeStorage ast_storage; + std::vector loaded_ast_uids; + std::vector>> + loaded_ops; + }; + cpp<#) + (:private + #>cpp + friend class boost::serialization::access; + + template + void serialize(TArchive &, const unsigned int) {} + cpp<#) + (:serialize :capnp :base t + :save-args '((helper "SaveHelper *")) + :load-args '((helper "LoadHelper *")))) + +#>cpp template std::pair, AstTreeStorage> LoadPlan( TArchive &ar) { @@ -235,6 +274,125 @@ cpp<# LoadPointers(${archive}, ${member-name}); cpp<#) +(defun capnp-save-vector (capnp-type cpp-type &optional lambda-code) + (let ((lambda-code (if lambda-code + lambda-code + "[](auto *builder, const auto &val) { val.Save(builder); }"))) + (lambda (builder member-name) + #>cpp + utils::SaveVector<${capnp-type}, ${cpp-type}>(${member-name}, &${builder}, ${lambda-code}); + cpp<#))) + +(defun capnp-load-vector (capnp-type cpp-type &optional lambda-code) + (let ((lambda-code (if lambda-code + lambda-code + (format nil + "[](const auto &reader) { ~A val; val.Load(reader); return val; }" + cpp-type)))) + (lambda (reader member-name) + #>cpp + utils::LoadVector<${capnp-type}, ${cpp-type}>(&${member-name}, ${reader}, ${lambda-code}); + cpp<#))) + +(defun capnp-save-optional (capnp-type cpp-type &optional lambda-code) + (let ((lambda-code (if lambda-code + lambda-code + "[](auto *builder, const auto &val) { val.Save(builder); }"))) + (lambda (builder member) + #>cpp + utils::SaveOptional<${capnp-type}, ${cpp-type}>(${member}, &${builder}, ${lambda-code}); + cpp<#))) + +(defun capnp-load-optional (capnp-type cpp-type &optional lambda-code) + (let ((lambda-code (if lambda-code + lambda-code + (format nil + "[](const auto &reader) { ~A val; val.Load(reader); return val; }" + cpp-type)))) + (lambda (reader member) + #>cpp + ${member} = utils::LoadOptional<${capnp-type}, ${cpp-type}>(${reader}, ${lambda-code}); + cpp<#))) + +(defun capnp-save-enum (capnp-type cpp-type enum-values) + (lambda (builder member) + (let* ((member-setter (remove #\_ (string-capitalize member))) + (cases (mapcar (lambda (value-symbol) + (let ((value (cl-ppcre:regex-replace-all "-" (string value-symbol) "_"))) + #>cpp + case ${cpp-type}::${value}: + ${builder}->set${member-setter}(${capnp-type}::${value}); + break; + cpp<#)) + enum-values))) + (format nil "switch (~A) {~%~{~A~%~}}" member (mapcar #'lcp::raw-cpp-string cases))))) + +(defun capnp-load-enum (capnp-type cpp-type enum-values) + (lambda (reader member) + (let* ((member-getter (remove #\_ (string-capitalize member))) + (cases (mapcar (lambda (value-symbol) + (let ((value (cl-ppcre:regex-replace-all "-" (string value-symbol) "_"))) + #>cpp + case ${capnp-type}::${value}: + ${member} = ${cpp-type}::${value}; + break; + cpp<#)) + enum-values))) + (format nil "switch (~A.get~A()) {~%~{~A~%~}}" + reader member-getter (mapcar #'lcp::raw-cpp-string cases))))) + +(defun save-ast-pointer (builder member) + (let ((member-getter (remove #\_ (string-capitalize member)))) + #>cpp + if (${member}) { + auto ${member}_builder = ${builder}->init${member-getter}(); + ${member}->Save(&${member}_builder, &helper->saved_ast_uids); + } + cpp<#)) + +(defun load-ast-pointer (ast-type) + (lambda (reader member) + (let ((member-getter (remove #\_ (string-capitalize member)))) + #>cpp + if (${reader}.has${member-getter}()) + ${member} = static_cast<${ast-type}>(helper->ast_storage.Load(${reader}.get${member-getter}(), &helper->loaded_ast_uids)); + else + ${member} = nullptr; + cpp<#))) + +(defun save-ast-vector (ast-type) + (capnp-save-vector "::query::capnp::Tree" ast-type + "[helper](auto *builder, const auto &val) { + val->Save(builder, &helper->saved_ast_uids); + }")) + +(defun load-ast-vector (ast-type) + (capnp-load-vector "::query::capnp::Tree" ast-type + (format + nil + "[helper](const auto &reader) { + // We expect the unsafe downcast via static_cast to work. + return static_cast<~A>(helper->ast_storage.Load(reader, &helper->loaded_ast_uids)); + }" ast-type))) + +(defun save-operator-pointer (builder member-name) + #>cpp + utils::SaveSharedPtr(${member-name}, &${builder}, + [helper](auto *builder, const auto &val) { + val.Save(builder, helper); + }, &helper->saved_ops); + cpp<#) + +(defun load-operator-pointer (reader member-name) + #>cpp + ${member-name} = utils::LoadSharedPtr(${reader}, + [helper](const auto &reader) { + auto op = LogicalOperator::Construct(reader); + op->Load(reader, helper); + return op.release(); + }, &helper->loaded_ops); + cpp<#) + (lcp:define-class once (logical-operator) () (:documentation @@ -265,11 +423,15 @@ and false on every following Pull.") bool did_pull_{false}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class create-node (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (node-atom "NodeAtom *" :initval "nullptr" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "NodeAtom *") :save-fun #'save-pointer :load-fun #'load-pointer) (on-random-worker :bool :initval "false")) (:documentation @@ -323,16 +485,22 @@ a preceeding `MATCH`), or multiple nodes (`MATCH ... CREATE` or const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class create-expand (logical-operator) ( ;; info on what's getting expanded - (node-atom "NodeAtom *" :save-fun #'save-pointer :load-fun #'load-pointer) - (edge-atom "EdgeAtom *" :save-fun #'save-pointer :load-fun #'load-pointer) + (node-atom "NodeAtom *" :save-fun #'save-pointer :load-fun #'load-pointer + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "NodeAtom *")) + (edge-atom "EdgeAtom *" :save-fun #'save-pointer :load-fun #'load-pointer + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "EdgeAtom *")) ;; the input op and the symbol under which the op's result ;; can be found in the frame - (input "std::shared_ptr") + (input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-symbol "Symbol") (existing-node :bool :documentation "if the given node atom refers to an existing node (either matched or created)")) @@ -409,12 +577,19 @@ chained in cases when longer paths need creating. ExpressionEvaluator &evaluator); }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class scan-all (logical-operator) - ((input "std::shared_ptr" :scope :protected) + ((input "std::shared_ptr" :scope :protected + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (output-symbol "Symbol" :reader t :scope :protected) (graph-view "GraphView" :reader t :scope :protected + :capnp-init nil + :capnp-save (capnp-save-enum "::query::capnp::GraphView" "GraphView" + '(old new)) + :capnp-load (capnp-load-enum "::query::capnp::GraphView" "GraphView" + '(old new)) :documentation "Controls which graph state is used to produce vertices. @@ -454,7 +629,7 @@ with a constructor argument. } cpp<#) (:protected #>cpp ScanAll() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class scan-all-by-label (scan-all) ((label "storage::Label" :reader t)) @@ -475,12 +650,39 @@ given label. database::GraphDbAccessor &db) const override; cpp<#) (:private #>cpp ScanAllByLabel() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) + +(defun save-optional-bound (builder member) + (let ((save-bound + "[helper](auto *builder, const auto &bound) { + builder->setType(bound.type() == Bound::Type::INCLUSIVE ? + ::utils::capnp::Bound<::query::capnp::Tree>::Type::INCLUSIVE : + ::utils::capnp::Bound<::query::capnp::Tree>::Type::EXCLUSIVE); + auto value_builder = builder->initValue(); + bound.value()->Save(&value_builder, &helper->saved_ast_uids); + }")) + (funcall (capnp-save-optional "::utils::capnp::Bound<::query::capnp::Tree>" "Bound" + save-bound) + builder member))) + +(defun load-optional-bound (reader member) + (let ((load-bound + "[helper](const auto &reader) { + auto type = reader.getType() == ::utils::capnp::Bound<::query::capnp::Tree>::Type::INCLUSIVE + ? Bound::Type::INCLUSIVE : Bound::Type::EXCLUSIVE; + auto *value = static_cast(helper->ast_storage.Load(reader.getValue(), &helper->loaded_ast_uids)); + return Bound(value, type); + }")) + (funcall (capnp-load-optional "::utils::capnp::Bound<::query::capnp::Tree>" "Bound" + load-bound) + reader member))) (lcp:define-class scan-all-by-label-property-range (scan-all) ((label "storage::Label" :reader t) (property "storage::Property" :reader t) (lower-bound "std::experimental::optional" :reader t + :capnp-type "Utils.Optional(Utils.Bound(Ast.Tree))" + :capnp-save #'save-optional-bound :capnp-load #'load-optional-bound :save-fun #>cpp auto save_bound = [&ar](auto &maybe_bound) { @@ -513,8 +715,11 @@ given label. load_bound(lower_bound_); cpp<#) (upper-bound "std::experimental::optional" :reader t + :capnp-save #'save-optional-bound :capnp-load #'load-optional-bound + :capnp-type "Utils.Optional(Utils.Bound(Ast.Tree))" :save-fun #>cpp save_bound(upper_bound_); cpp<# - :load-fun #>cpp load_bound(upper_bound_); cpp<#)) + :load-fun #>cpp load_bound(upper_bound_); cpp<# + )) (:documentation "Behaves like @c ScanAll, but produces only vertices with given label and property value which is inside a range (inclusive or exlusive). @@ -552,12 +757,14 @@ property value which is inside a range (inclusive or exlusive). database::GraphDbAccessor &db) const override; cpp<#) (:private #>cpp ScanAllByLabelPropertyRange() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class scan-all-by-label-property-value (scan-all) ((label "storage::Label" :reader t) (property "storage::Property" :reader t) (expression "Expression *" :reader t + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Behaves like @c ScanAll, but produces only vertices with given label and @@ -589,23 +796,38 @@ property value. database::GraphDbAccessor &db) const override; cpp<#) (:private #>cpp ScanAllByLabelPropertyValue() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class expand-common () ( ;; info on what's getting expanded (node-symbol "Symbol" :reader t :scope :protected) (edge-symbol "Symbol" :reader t :scope :protected) - (direction "EdgeAtom::Direction" :reader t :scope :protected) - (edge-types "std::vector" :reader t :scope :protected) + (direction "EdgeAtom::Direction" :reader t :scope :protected + :capnp-type "Ast.EdgeAtom.Direction" :capnp-init nil + :capnp-save (capnp-save-enum "::query::capnp::EdgeAtom::Direction" "EdgeAtom::Direction" + '(in out both)) + :capnp-load (capnp-load-enum "::query::capnp::EdgeAtom::Direction" "EdgeAtom::Direction" + '(in out both))) + (edge-types "std::vector" :reader t :scope :protected + :capnp-save (capnp-save-vector "::storage::capnp::Common" "storage::EdgeType") + :capnp-load (capnp-load-vector "::storage::capnp::Common" "storage::EdgeType")) ;; the input op and the symbol under which the op's result ;; can be found in the frame - (input "std::shared_ptr" :scope :protected) + (input "std::shared_ptr" :scope :protected + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-symbol "Symbol" :reader t :scope :protected) (existing-node :bool :scope :protected :documentation "If the given node atom refer to a symbol that has already been expanded and should be just validated in the frame.") - (graph-view "GraphView" :reader t :scope :protected :documentation + (graph-view "GraphView" :reader t :scope :protected + :capnp-init nil + :capnp-save (capnp-save-enum "::query::capnp::GraphView" "GraphView" + '(old new)) + :capnp-load (capnp-load-enum "::query::capnp::GraphView" "GraphView" + '(old new)) + :documentation "from which state the input node should get expanded")) (:documentation "Common functionality and data members of single-edge and variable-length @@ -649,6 +871,8 @@ expansion") cpp<#) (:protected #>cpp + virtual ~ExpandCommon() {} + // types that we'll use for members in both subclasses using InEdgeT = decltype(std::declval().in()); using InEdgeIteratorT = decltype(std::declval().in().begin()); @@ -668,7 +892,9 @@ expansion") ExpandCommon() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp + :save-args '((helper "LogicalOperator::SaveHelper *")) + :load-args '((helper "LogicalOperator::LoadHelper *")))) (lcp:define-class expand (logical-operator expand-common) () @@ -738,19 +964,36 @@ pulled.") bool InitEdges(Frame &, Context &); }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp :inherit-compose '(expand-common))) (lcp:define-class expand-variable (logical-operator expand-common) - ((type "EdgeAtom::Type" :reader t) + ((type "EdgeAtom::Type" :reader t :capnp-type "Ast.EdgeAtom.Type" + :capnp-init nil + :capnp-save (capnp-save-enum "::query::capnp::EdgeAtom::Type" "EdgeAtom::Type" + '(single depth-first breadth-first weighted-shortest-path)) + :capnp-load (capnp-load-enum "::query::capnp::EdgeAtom::Type" "EdgeAtom::Type" + '(single depth-first breadth-first weighted-shortest-path))) (is-reverse :bool :documentation "True if the path should be written as expanding from node_symbol to input_symbol.") (lower-bound "Expression *" :save-fun #'save-pointer :load-fun #'load-pointer + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :documentation "Optional lower bound of the variable length expansion, defaults are (1, inf)") (upper-bound "Expression *" :save-fun #'save-pointer :load-fun #'load-pointer + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :documentation "Optional upper bound of the variable length expansion, defaults are (1, inf)") (filter-lambda "Lambda") - (weight-lambda "std::experimental::optional") - (total-weight "std::experimental::optional")) + (weight-lambda "std::experimental::optional" + :capnp-save (capnp-save-optional + "capnp::ExpandVariable::Lambda" "Lambda" + "[helper](auto *builder, const auto &val) { val.Save(builder, helper); }") + :capnp-load (capnp-load-optional + "capnp::ExpandVariable::Lambda" "Lambda" + "[helper](const auto &reader) { Lambda val; val.Load(reader, helper); return val; }")) + (total-weight "std::experimental::optional" + :capnp-save (capnp-save-optional "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-optional "::query::capnp::Symbol" "Symbol"))) (:documentation "Variable-length expansion operator. For a node existing in the frame it expands a variable number of edges and places them @@ -773,8 +1016,12 @@ pulled.") ((inner-edge-symbol "Symbol" :documentation "Currently expanded edge symbol.") (inner-node-symbol "Symbol" :documentation "Currently expanded node symbol.") (expression "Expression *" :documentation "Expression used in lambda during expansion." + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) - (:serialize :boost)) + (:serialize :boost :capnp + :save-args '((helper "LogicalOperator::SaveHelper *")) + :load-args '((helper "LogicalOperator::LoadHelper *")))) #>cpp /** * Creates a variable-length expansion. Most params are forwarded @@ -797,7 +1044,7 @@ pulled.") * assigned to this symbol so it can be evaulated by the 'where' * expression. * @param filter_ The filter that must be satisfied for an expansion to - * succeed. Can use inner(node|edge) symbols. If nullptr, it is ignored. + * succeed. Can use inner(node/edge) symbols. If nullptr, it is ignored. */ ExpandVariable(Symbol node_symbol, Symbol edge_symbol, EdgeAtom::Type type, EdgeAtom::Direction direction, @@ -833,12 +1080,16 @@ pulled.") ExpandVariable() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp :inherit-compose '(expand-common))) (lcp:define-class construct-named-path (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (path-symbol "Symbol" :reader t) - (path-elements "std::vector" :reader t)) + (path-elements "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Constructs a named path from it's elements and places it on the frame.") (:public @@ -862,11 +1113,15 @@ pulled.") cpp<#) (:private #>cpp ConstructNamedPath() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class filter (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (expression "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Filter whose Pull returns true only when the given expression @@ -905,11 +1160,16 @@ a boolean value.") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class produce (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (named-expressions "std::vector" :reader t + :capnp-type "List(Ast.Tree)" + :capnp-save (save-ast-vector "NamedExpression *") + :capnp-load (load-ast-vector "NamedExpression *") :save-fun #'save-pointers :load-fun #'load-pointers)) (:documentation "A logical operator that places an arbitrary number @@ -953,11 +1213,16 @@ RETURN clause) the Produce's pull succeeds exactly once.") Produce() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class delete (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (expressions "std::vector" + :capnp-type "List(Ast.Tree)" + :capnp-save (save-ast-vector "Expression *") + :capnp-load (load-ast-vector "Expression *") :save-fun #'save-pointers :load-fun #'load-pointers) (detach :bool :documentation "if the vertex should be detached before deletion if not detached, @@ -997,13 +1262,19 @@ Has a flag for using DETACH DELETE when deleting vertices.") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class set-property (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (lhs "PropertyLookup *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "PropertyLookup *") :save-fun #'save-pointer :load-fun #'load-pointer) (rhs "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Logical Op for setting a single property on a single vertex or edge. @@ -1041,37 +1312,38 @@ can be stored (a TypedValue that can be converted to PropertyValue).") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class set-properties (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-symbol "Symbol") (rhs "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer) - (op "Op")) + (op "Op" :capnp-init nil + :capnp-save (capnp-save-enum "capnp::SetProperties::Op" "Op" '(update replace)) + :capnp-load (capnp-load-enum "capnp::SetProperties::Op" "Op" '(update replace)))) (:documentation - "Logical op for setting the whole properties set on a vertex or an -edge. + "Logical op for setting the whole properties set on a vertex or an edge. The value being set is an expression that must evaluate to a vertex, edge -or -map (literal or parameter). +or map (literal or parameter). Supports setting (replacing the whole properties set with another) and updating.") (:public - #>cpp - /** - * @brief Defines how setting the properties works. - * - * @c UPDATE means that the current property set is augmented with - * additional - * ones (existing props of the same name are replaced), while @c REPLACE - * means - * that the old props are discarded and replaced with new ones. - */ - enum class Op { UPDATE, REPLACE }; + (lcp:define-enum op + "Defines how setting the properties works. +@c UPDATE means that the current property set is augmented with additional +ones (existing props of the same name are replaced), while @c REPLACE means +that the old props are discarded and replaced with new ones." + update replace) + + #>cpp SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1110,12 +1382,16 @@ updating.") void Set(TRecordAccessor &record, const TypedValue &rhs) const; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class set-labels (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-symbol "Symbol") - (labels "std::vector")) + (labels "std::vector" + :capnp-save (capnp-save-vector "::storage::capnp::Common" "storage::Label") + :capnp-load (capnp-load-vector "::storage::capnp::Common" "storage::Label"))) (:documentation "Logical operator for setting an arbitrary number of labels on a Vertex. @@ -1150,11 +1426,15 @@ It does NOT remove labels that are already set on that Vertex.") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class remove-property (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (lhs "PropertyLookup *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "PropertyLookup *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Logical op for removing a property from an edge or a vertex.") @@ -1190,12 +1470,16 @@ It does NOT remove labels that are already set on that Vertex.") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class remove-labels (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-symbol "Symbol") - (labels "std::vector")) + (labels "std::vector" + :capnp-save (capnp-save-vector "::storage::capnp::Common" "storage::Label") + :capnp-load (capnp-load-vector "::storage::capnp::Common" "storage::Label"))) (:documentation "Logical operator for removing an arbitrary number of labels on a Vertex. @@ -1230,18 +1514,22 @@ If a label does not exist on a Vertex, nothing happens.") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class (expand-uniqueness-filter t-accessor) (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (expand-symbol "Symbol") - (previous-symbols "std::vector")) + (previous-symbols "std::vector" + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Filter whose Pull returns true only when the given expand_symbol frame value (the latest expansion) is not equal to any of the previous_symbols frame values. -Used for implementing [iso|cypher]morphism. +Used for implementing [iso/cypher]morphism. Isomorphism is vertex-uniqueness. It means that two different vertices in a pattern can not map to the same data vertex. For example, if the database @@ -1290,11 +1578,15 @@ between edges and an edge lists).") const std::unique_ptr input_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp :type-args '(vertex-accessor edge-accessor))) (lcp:define-class accumulate (logical-operator) - ((input "std::shared_ptr") - (symbols "std::vector" :reader t) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol")) (advance-command :bool :reader t)) (:documentation "Pulls everything from the input before passing it through. @@ -1357,7 +1649,7 @@ has been cached will be reconstructed before Pull returns. bool pulled_all_input_{false}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) #>cpp /** @@ -1371,11 +1663,24 @@ struct TypedValueVectorEqual { cpp<# (lcp:define-class aggregate (logical-operator) - ((input "std::shared_ptr") - (aggregations "std::vector" :reader t) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (aggregations "std::vector" :reader t + :capnp-save (capnp-save-vector + "capnp::Aggregate::Element" "Element" + "[helper](auto *builder, const auto &val) { val.Save(builder, helper); }") + :capnp-load (capnp-load-vector + "capnp::Aggregate::Element" "Element" + "[helper](const auto &reader) { Element val; val.Load(reader, helper); return val; }")) (group-by "std::vector" :reader t + :capnp-type "List(Ast.Tree)" + :capnp-save (save-ast-vector "Expression *") + :capnp-load (load-ast-vector "Expression *") :save-fun #'save-pointers :load-fun #'load-pointers) - (remember "std::vector" :reader t)) + (remember "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Performs an arbitrary number of aggregations of data from the given input grouped by the given criteria. @@ -1393,16 +1698,26 @@ elements are in an undefined state after aggregation.") (:public (lcp:define-struct element () ((value "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer) (key "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer) - (op "Aggregation::Op") - (output_sym "Symbol")) + (op "Aggregation::Op" :capnp-type "Ast.Aggregation.Op" + :capnp-init nil :capnp-save (capnp-save-enum "::query::capnp::Aggregation::Op" "Aggregation::Op" + '(count min max sum avg collect-list collect-map)) + :capnp-load (capnp-load-enum "::query::capnp::Aggregation::Op" "Aggregation::Op" + '(count min max sum avg collect-list collect-map))) + (output-sym "Symbol")) (:documentation "An aggregation element, contains: (input data expression, key expression - only used in COLLECT_MAP, type of aggregation, output symbol).") - (:serialize :boost)) + (:serialize :boost :capnp + :save-args '((helper "LogicalOperator::SaveHelper *")) + :load-args '((helper "LogicalOperator::LoadHelper *")))) #>cpp Aggregate(const std::shared_ptr &input, const std::vector &aggregations, @@ -1503,11 +1818,15 @@ elements are in an undefined state after aggregation.") void EnsureOkForAvgSum(const TypedValue &value) const; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class skip (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (expression "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Skips a number of Pulls from the input op. @@ -1556,11 +1875,15 @@ operator's implementation does not expect this.") int skipped_{0}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class limit (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (expression "Expression *" + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer)) (:documentation "Limits the number of Pulls from the input op. @@ -1612,14 +1935,22 @@ input should be performed).") int pulled_{0}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class order-by (logical-operator) - ((input "std::shared_ptr") - (compare "TypedValueVectorCompare" :reader t) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (compare "TypedValueVectorCompare" :reader t + :capnp-type "Common.TypedValueVectorCompare") (order-by "std::vector" :reader t + :capnp-type "List(Ast.Tree)" + :capnp-save (save-ast-vector "Expression *") + :capnp-load (load-ast-vector "Expression *") :save-fun #'save-pointers :load-fun #'load-pointers) - (output-symbols "std::vector" :reader t)) + (output-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Logical operator for ordering (sorting) results. @@ -1673,12 +2004,18 @@ are valid for usage after the OrderBy operator.") decltype(cache_.begin()) cache_it_ = cache_.begin(); }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class merge (logical-operator) - ((input "std::shared_ptr") - (merge-match "std::shared_ptr" :reader t) - (merge-create "std::shared_ptr" :reader t)) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (merge-match "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (merge-create "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer)) (:documentation "Merge operator. For every sucessful Pull from the input operator a Pull from the merge_match is attempted. All @@ -1733,12 +2070,18 @@ documentation.") bool pull_input_{true}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class optional (logical-operator) - ((input "std::shared_ptr") - (optional "std::shared_ptr" :reader t) - (optional-symbols "std::vector" :reader t)) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (optional "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (optional-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Optional operator. Used for optional match. For every successful Pull from the input branch a Pull from the optional @@ -1785,11 +2128,15 @@ and returns true, once.") bool pull_input_{true}; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class unwind (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (input-expression "Expression *" :reader t + :capnp-type "Ast.Tree" :capnp-init nil + :capnp-save #'save-ast-pointer :capnp-load (load-ast-pointer "Expression *") :save-fun #'save-pointer :load-fun #'load-pointer) (output-symbol "Symbol")) (:documentation @@ -1832,11 +2179,15 @@ Input is optional (unwind can be the first clause in a query).") std::vector::iterator input_value_it_ = input_value_.end(); }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class distinct (logical-operator) - ((input "std::shared_ptr") - (value-symbols "std::vector")) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (value-symbols "std::vector" + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Ensures that only distinct rows are yielded. This implementation accepts a vector of Symbols @@ -1884,7 +2235,7 @@ This implementation maintains input ordering.") seen_rows_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class create-index (logical-operator) ((label "storage::Label" :reader t) @@ -1911,14 +2262,24 @@ case the index already exists, nothing happens.") void set_input(std::shared_ptr) override; cpp<#) (:private #>cpp CreateIndex() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class union (logical-operator) - ((left-op "std::shared_ptr") - (right-op "std::shared_ptr") - (union-symbols "std::vector") - (left-symbols "std::vector") - (right-symbols "std::vector")) + ((left-op "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (right-op "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (union-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol")) + (left-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol")) + (right-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "A logical operator that applies UNION operator on inputs and places the result on the frame. @@ -1957,12 +2318,16 @@ vectors of symbols used by each of the inputs.") const std::unique_ptr left_cursor_, right_cursor_; }; cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class pull-remote (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (plan-id :int64_t :initval 0 :reader t) - (symbols "std::vector" :reader t)) + (symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "An operator in distributed Memgraph that yields both local and remote (from other workers) frames. Obtaining remote frames is done through RPC calls to @@ -1988,11 +2353,26 @@ time on data transfer. It gives no guarantees on result order.") } cpp<#) (:private #>cpp PullRemote() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) + +(defun load-pull-remote (reader member-name) + #>cpp + ${member-name} = std::static_pointer_cast( + utils::LoadSharedPtr(${reader}, + [helper](const auto &reader) { + auto op = LogicalOperator::Construct(reader); + op->Load(reader, helper); + return op.release(); + }, &helper->loaded_ops)); + cpp<#) (lcp:define-class synchronize (logical-operator) - ((input "std::shared_ptr") - (pull-remote "std::shared_ptr" :reader t) + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (pull-remote "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-pull-remote) (advance-command :bool :initval "false" :reader t)) (:documentation "Operator used to synchronize stages of plan execution between the master and @@ -2037,13 +2417,21 @@ Logic of the synchronize operator is: } cpp<#) (:private #>cpp Synchronize() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class cartesian (logical-operator) - ((left-op "std::shared_ptr" :reader t) - (left-symbols "std::vector" :reader t) - (right-op "std::shared_ptr" :reader t) - (right-symbols "std::vector" :reader t)) + ((left-op "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (left-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol")) + (right-op "std::shared_ptr" :reader t + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) + (right-symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol"))) (:documentation "Operator for producing a Cartesian product from 2 input branches") (:public @@ -2068,15 +2456,23 @@ Logic of the synchronize operator is: void set_input(std::shared_ptr) override; cpp<#) (:private #>cpp Cartesian() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) (lcp:define-class pull-remote-order-by (logical-operator) - ((input "std::shared_ptr") + ((input "std::shared_ptr" + :capnp-save #'save-operator-pointer + :capnp-load #'load-operator-pointer) (plan-id :int64_t :initval 0 :reader t) - (symbols "std::vector" :reader t) + (symbols "std::vector" :reader t + :capnp-save (capnp-save-vector "::query::capnp::Symbol" "Symbol") + :capnp-load (capnp-load-vector "::query::capnp::Symbol" "Symbol")) (order-by "std::vector" :reader t + :capnp-type "List(Ast.Tree)" + :capnp-save (save-ast-vector "Expression *") + :capnp-load (load-ast-vector "Expression *") :save-fun #'save-pointers :load-fun #'load-pointers) - (compare "TypedValueVectorCompare" :reader t)) + (compare "TypedValueVectorCompare" :reader t + :capnp-type "Common.TypedValueVectorCompare")) (:documentation "Operator that merges distributed OrderBy operators. Instead of using a regular OrderBy on master (which would collect all remote @@ -2103,13 +2499,12 @@ by having only one result from each worker.") } cpp<#) (:private #>cpp PullRemoteOrderBy() {} cpp<#) - (:serialize :boost)) + (:serialize :boost :capnp)) + +(lcp:pop-namespace) ;; plan +(lcp:pop-namespace) ;; query #>cpp - -} // namespace plan -} // namespace query - BOOST_CLASS_EXPORT_KEY(query::plan::Once); BOOST_CLASS_EXPORT_KEY(query::plan::CreateNode); BOOST_CLASS_EXPORT_KEY(query::plan::CreateExpand); diff --git a/src/storage/types.hpp b/src/storage/types.hpp index e25b3b89f..6121a43ce 100644 --- a/src/storage/types.hpp +++ b/src/storage/types.hpp @@ -56,11 +56,11 @@ class Common : public utils::TotalOrdering { size_t operator()(const TSpecificType &t) const { return hash(t.id_); } }; - virtual void Save(capnp::Common::Builder &builder) const { - builder.setStorage(id_); + virtual void Save(capnp::Common::Builder *builder) const { + builder->setStorage(id_); } - virtual void Load(capnp::Common::Reader &reader) { + virtual void Load(const capnp::Common::Reader &reader) { id_ = reader.getStorage(); } diff --git a/src/utils/serialization.capnp b/src/utils/serialization.capnp index 98468419b..8dfb73cd2 100644 --- a/src/utils/serialization.capnp +++ b/src/utils/serialization.capnp @@ -62,7 +62,8 @@ struct SharedPtr(T) { } } -# TypedValue. +# Our types + struct TypedValue { union { nullType @0 :Void; @@ -82,3 +83,13 @@ struct TypedValue { value @1 :TypedValue; } } + +struct Bound(T) { + type @0 :Type; + value @1 :T; + + enum Type { + inclusive @0; + exclusive @1; + } +} diff --git a/src/utils/serialization.hpp b/src/utils/serialization.hpp index 75ea3b686..fb8d51e02 100644 --- a/src/utils/serialization.hpp +++ b/src/utils/serialization.hpp @@ -203,13 +203,13 @@ inline void SaveUniquePtr( template inline std::unique_ptr LoadUniquePtr( const typename capnp::UniquePtr::Reader &reader, - const std::function &load) { + const std::function &load) { switch (reader.which()) { case capnp::UniquePtr::NULLPTR: return nullptr; case capnp::UniquePtr::VALUE: auto value_reader = reader.getValue(); - return std::make_unique(load(value_reader)); + return std::unique_ptr(load(value_reader)); } } @@ -236,9 +236,9 @@ inline void SaveSharedPtr( } template -inline std::shared_ptr LoadSharedPtr( +std::shared_ptr LoadSharedPtr( const typename capnp::SharedPtr::Reader &reader, - const std::function &load, + const std::function &load, std::vector>> *loaded_pointers) { std::shared_ptr ret; switch (reader.which()) { @@ -255,7 +255,7 @@ inline std::shared_ptr LoadSharedPtr( }); if (found != loaded_pointers->end()) return found->second; auto value_reader = entry_reader.getValue(); - ret = std::make_shared(load(value_reader)); + ret = std::shared_ptr(load(value_reader)); loaded_pointers->emplace_back(std::make_pair(pointer_id, ret)); } return ret; diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index 26dd1d9a8..1e2abf72b 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -19,7 +19,6 @@ #include "query/typed_value.hpp" #include "capnp/message.h" -#include "capnp/serialize-packed.h" #include "query/frontend/ast/ast.capnp.h" namespace { diff --git a/tests/unit/query_planner.cpp b/tests/unit/query_planner.cpp index 179ed2d3d..eccd344de 100644 --- a/tests/unit/query_planner.cpp +++ b/tests/unit/query_planner.cpp @@ -17,6 +17,9 @@ #include "query/plan/operator.hpp" #include "query/plan/planner.hpp" +#include "capnp/message.h" +#include "query/plan/operator.capnp.h" + #include "query_common.hpp" namespace query { @@ -505,6 +508,42 @@ class SerializedPlanner { std::unique_ptr plan_; }; +void SavePlan(const LogicalOperator &plan, ::capnp::MessageBuilder *message) { + auto builder = message->initRoot(); + LogicalOperator::SaveHelper helper; + plan.Save(&builder, &helper); +} + +auto LoadPlan(const ::query::plan::capnp::LogicalOperator::Reader &reader) { + auto plan = LogicalOperator::Construct(reader); + LogicalOperator::LoadHelper helper; + plan->Load(reader, &helper); + return std::make_pair(std::move(plan), std::move(helper.ast_storage)); +} + +class CapnpPlanner { + public: + CapnpPlanner(std::vector single_query_parts, + PlanningContext &context) { + ::capnp::MallocMessageBuilder message; + { + auto original_plan = MakeLogicalPlanForSingleQuery( + single_query_parts, context); + SavePlan(*original_plan, &message); + } + { + auto reader = message.getRoot(); + std::tie(plan_, ast_storage_) = LoadPlan(reader); + } + } + + auto &plan() { return *plan_; } + + private: + AstTreeStorage ast_storage_; + std::unique_ptr plan_; +}; + template TPlanner MakePlanner(database::MasterBase &master_db, AstTreeStorage &storage, SymbolTable &symbol_table) { @@ -637,7 +676,7 @@ ExpectedDistributedPlan ExpectDistributed( template class TestPlanner : public ::testing::Test {}; -using PlannerTypes = ::testing::Types; +using PlannerTypes = ::testing::Types; TYPED_TEST_CASE(TestPlanner, PlannerTypes); @@ -2234,4 +2273,111 @@ TYPED_TEST(TestPlanner, DistributedCartesianCreate) { CheckDistributedPlan(planner.plan(), symbol_table, expected); } +TEST(CapnpSerial, Union) { + std::vector left_symbols{ + Symbol("symbol", 1, true, Symbol::Type::Edge)}; + std::vector right_symbols{ + Symbol("symbol", 3, true, Symbol::Type::Any)}; + auto union_symbols = right_symbols; + auto union_op = std::make_unique(nullptr, nullptr, union_symbols, + left_symbols, right_symbols); + std::unique_ptr loaded_plan; + ::capnp::MallocMessageBuilder message; + SavePlan(*union_op, &message); + AstTreeStorage new_storage; + std::tie(loaded_plan, new_storage) = + LoadPlan(message.getRoot()); + ASSERT_TRUE(loaded_plan); + auto *loaded_op = dynamic_cast(loaded_plan.get()); + ASSERT_TRUE(loaded_op); + EXPECT_FALSE(loaded_op->left_op()); + EXPECT_FALSE(loaded_op->right_op()); + EXPECT_EQ(loaded_op->left_symbols(), left_symbols); + EXPECT_EQ(loaded_op->right_symbols(), right_symbols); + EXPECT_EQ(loaded_op->union_symbols(), union_symbols); +} + +TEST(CapnpSerial, Cartesian) { + std::vector left_symbols{ + Symbol("left_symbol", 1, true, Symbol::Type::Edge)}; + std::vector right_symbols{ + Symbol("right_symbol", 3, true, Symbol::Type::Any)}; + auto cartesian = std::make_unique(nullptr, left_symbols, nullptr, + right_symbols); + std::unique_ptr loaded_plan; + ::capnp::MallocMessageBuilder message; + SavePlan(*cartesian, &message); + AstTreeStorage new_storage; + std::tie(loaded_plan, new_storage) = + LoadPlan(message.getRoot()); + ASSERT_TRUE(loaded_plan); + auto *loaded_op = dynamic_cast(loaded_plan.get()); + ASSERT_TRUE(loaded_op); + EXPECT_FALSE(loaded_op->left_op()); + EXPECT_FALSE(loaded_op->right_op()); + EXPECT_EQ(loaded_op->left_symbols(), left_symbols); + EXPECT_EQ(loaded_op->right_symbols(), right_symbols); +} + +TEST(CapnpSerial, Synchronize) { + auto synchronize = std::make_unique(nullptr, nullptr, true); + std::unique_ptr loaded_plan; + ::capnp::MallocMessageBuilder message; + SavePlan(*synchronize, &message); + AstTreeStorage new_storage; + std::tie(loaded_plan, new_storage) = + LoadPlan(message.getRoot()); + ASSERT_TRUE(loaded_plan); + auto *loaded_op = dynamic_cast(loaded_plan.get()); + ASSERT_TRUE(loaded_op); + EXPECT_FALSE(loaded_op->input()); + EXPECT_FALSE(loaded_op->pull_remote()); + EXPECT_TRUE(loaded_op->advance_command()); +} + +TEST(CapnpSerial, PullRemote) { + std::vector symbols{ + Symbol("symbol", 1, true, Symbol::Type::Edge)}; + auto pull_remote = std::make_unique(nullptr, 42, symbols); + std::unique_ptr loaded_plan; + ::capnp::MallocMessageBuilder message; + SavePlan(*pull_remote, &message); + AstTreeStorage new_storage; + std::tie(loaded_plan, new_storage) = + LoadPlan(message.getRoot()); + ASSERT_TRUE(loaded_plan); + auto *loaded_op = dynamic_cast(loaded_plan.get()); + ASSERT_TRUE(loaded_op); + EXPECT_FALSE(loaded_op->input()); + EXPECT_EQ(loaded_op->plan_id(), 42); + EXPECT_EQ(loaded_op->symbols(), symbols); +} + +TEST(CapnpSerial, PullRemoteOrderBy) { + auto once = std::make_shared(); + AstTreeStorage storage; + std::vector symbols{ + Symbol("my_symbol", 2, true, Symbol::Type::Vertex, 3)}; + std::vector> order_by{ + {query::Ordering::ASC, IDENT("my_symbol")}}; + auto pull_remote_order_by = + std::make_unique(once, 42, order_by, symbols); + std::unique_ptr loaded_plan; + ::capnp::MallocMessageBuilder message; + SavePlan(*pull_remote_order_by, &message); + AstTreeStorage new_storage; + std::tie(loaded_plan, new_storage) = + LoadPlan(message.getRoot()); + ASSERT_TRUE(loaded_plan); + auto *loaded_op = dynamic_cast(loaded_plan.get()); + ASSERT_TRUE(loaded_op); + ASSERT_TRUE(std::dynamic_pointer_cast(loaded_op->input())); + EXPECT_EQ(loaded_op->plan_id(), 42); + EXPECT_EQ(loaded_op->symbols(), symbols); + ASSERT_EQ(loaded_op->order_by().size(), 1); + EXPECT_TRUE(dynamic_cast(loaded_op->order_by()[0])); + ASSERT_EQ(loaded_op->compare().ordering().size(), 1); + EXPECT_EQ(loaded_op->compare().ordering()[0], query::Ordering::ASC); +} + } // namespace diff --git a/tests/unit/serialization.cpp b/tests/unit/serialization.cpp index c13cbba8e..83ab7f6e8 100644 --- a/tests/unit/serialization.cpp +++ b/tests/unit/serialization.cpp @@ -91,7 +91,6 @@ void CheckOptionalInt(const std::experimental::optional &x1) { TEST(Serialization, CapnpOptional) { std::experimental::optional x1 = {}; std::experimental::optional x2 = 42; - std::experimental::optional y1, y2; CheckOptionalInt(x1); CheckOptionalInt(x2); @@ -104,13 +103,11 @@ TEST(Serialization, CapnpOptionalNonCopyable) { { auto builder = message.initRoot>>(); - auto save = []( - utils::capnp::UniquePtr::Builder *builder, - const std::unique_ptr &data) { - auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) { - builder->setValue(value); + auto save = [](auto *ptr_builder, const auto &data) { + auto save_int = [](auto *int_builder, int value) { + int_builder->setValue(value); }; - utils::SaveUniquePtr(data, builder, + utils::SaveUniquePtr(data, ptr_builder, save_int); }; utils::SaveOptional, @@ -120,13 +117,11 @@ TEST(Serialization, CapnpOptionalNonCopyable) { { auto reader = message.getRoot>>(); - auto load = []( - const utils::capnp::UniquePtr::Reader &reader) - -> std::unique_ptr { - auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &ptr_reader) { + auto load_int = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; - return utils::LoadUniquePtr(reader, + return utils::LoadUniquePtr(ptr_reader, load_int); }; element = @@ -150,8 +145,8 @@ void CheckUniquePtrInt(const std::unique_ptr &x1) { { auto reader = message.getRoot>(); - auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; y1 = utils::LoadUniquePtr(reader, load); } @@ -176,13 +171,11 @@ TEST(Serialization, CapnpUniquePtrNonCopyable) { { auto builder = message.initRoot>>(); - auto save = []( - utils::capnp::UniquePtr::Builder *builder, - const std::unique_ptr &data) { - auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) { - builder->setValue(value); + auto save = [](auto *ptr_builder, const auto &data) { + auto save_int = [](auto *int_builder, int value) { + int_builder->setValue(value); }; - utils::SaveUniquePtr(data, builder, + utils::SaveUniquePtr(data, ptr_builder, save_int); }; utils::SaveUniquePtr, @@ -192,14 +185,13 @@ TEST(Serialization, CapnpUniquePtrNonCopyable) { { auto reader = message.getRoot>>(); - auto load = []( - const utils::capnp::UniquePtr::Reader &reader) - -> std::unique_ptr { - auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &ptr_reader) { + auto load_int = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; - return utils::LoadUniquePtr(reader, - load_int); + return new std::unique_ptr( + utils::LoadUniquePtr(ptr_reader, + load_int)); }; element = utils::LoadUniquePtr, @@ -233,8 +225,8 @@ TEST(Serialization, CapnpSharedPtr) { { auto reader = message.getRoot< ::capnp::List>>(); - auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; for (const auto ptr_reader : reader) { elements.emplace_back(utils::LoadSharedPtr( @@ -256,13 +248,11 @@ TEST(Serialization, CapnpSharedPtrNonCopyable) { { auto builder = message.initRoot>>(); - auto save = []( - utils::capnp::UniquePtr::Builder *builder, - const std::unique_ptr &data) { - auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) { - builder->setValue(value); + auto save = [](auto *ptr_builder, const auto &data) { + auto save_int = [](auto *int_builder, int value) { + int_builder->setValue(value); }; - utils::SaveUniquePtr(data, builder, + utils::SaveUniquePtr(data, ptr_builder, save_int); }; utils::SaveSharedPtr, @@ -275,14 +265,13 @@ TEST(Serialization, CapnpSharedPtrNonCopyable) { { auto reader = message.getRoot>>(); - auto load = []( - const utils::capnp::UniquePtr::Reader &reader) - -> std::unique_ptr { - auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &ptr_reader) { + auto load_int = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; - return utils::LoadUniquePtr(reader, - load_int); + return new std::unique_ptr( + utils::LoadUniquePtr(ptr_reader, + load_int)); }; element = utils::LoadSharedPtr, @@ -344,13 +333,11 @@ TEST(Serialization, CapnpVectorNonCopyable) { auto list_builder = message.initRoot< ::capnp::List>>( data.size()); - auto save = []( - utils::capnp::UniquePtr::Builder *builder, - const std::unique_ptr &data) { - auto save_int = [](utils::capnp::BoxInt32::Builder *builder, int value) { - builder->setValue(value); + auto save = [](auto *ptr_builder, const auto &data) { + auto save_int = [](auto *int_builder, int value) { + int_builder->setValue(value); }; - utils::SaveUniquePtr(data, builder, + utils::SaveUniquePtr(data, ptr_builder, save_int); }; utils::SaveVector, @@ -360,13 +347,11 @@ TEST(Serialization, CapnpVectorNonCopyable) { { auto reader = message.getRoot< ::capnp::List>>(); - auto load = []( - const utils::capnp::UniquePtr::Reader &reader) - -> std::unique_ptr { - auto load_int = [](const utils::capnp::BoxInt32::Reader &reader) -> int { - return reader.getValue(); + auto load = [](const auto &ptr_reader) { + auto load_int = [](const auto &int_reader) { + return new int(int_reader.getValue()); }; - return utils::LoadUniquePtr(reader, + return utils::LoadUniquePtr(ptr_reader, load_int); }; utils::LoadVector, diff --git a/tools/lcp b/tools/lcp index 621f3c29e..38d40a07a 100755 --- a/tools/lcp +++ b/tools/lcp @@ -1,8 +1,9 @@ #!/bin/bash -e -if [[ $# -ne 1 ]]; then - echo "Usage: $0 LCP_FILE" - echo "Convert a LCP_FILE to C++ header file and output to stdout" +if [[ $# -ne 1 && $# -ne 2 ]]; then + echo "Usage: $0 LCP_FILE [CAPNP_EXE]" + echo "Convert a LCP_FILE to C++ header file and output to stdout." + echo "If CAPNP_EXE is provided, then the Cap'n Proto schema generated." exit 1 fi @@ -14,17 +15,28 @@ fi lcp_file=`realpath $1` script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $script_dir - quicklisp_install_dir="$HOME/quicklisp" if [[ -v QUICKLISP_HOME ]]; then quicklisp_install_dir="${QUICKLISP_HOME}" fi +capnp="" +if [[ $# -eq 2 ]]; then + capnp=":capnp-id \"$($2 id)\"" +fi + echo \ " (load \"${quicklisp_install_dir}/setup.lisp\") (ql:quickload :cl-ppcre :silent t) -(load \"../src/lisp/lcp.lisp\") -(lcp:process-file \"$lcp_file\" :out-stream t) -" | sbcl --script | clang-format -style=file +(load \"$script_dir/../src/lisp/lcp.lisp\") +(lcp:process-file \"$lcp_file\" $capnp) +" | sbcl --script + +filename=`basename $lcp_file .lcp` +hpp_file="$(dirname $lcp_file)/$filename.hpp" +clang-format -style=file -i $hpp_file + +if [[ $# -eq 2 ]]; then + clang-format -style=file -i "$lcp_file.cpp" +fi