tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add basic support for saving classes with SLK

Browse Source 
Summary:
This is the initial work on transferring our serialization code from
Cap'n Proto to SaveLoadKit. The commit contains tests for generated
code, but still requires work on supporting some features. Most notably,
generating and storing type IDs for derived classes so that they can be
loaded from a base pointer. Naturally, loading implementation hasn't
even been started yet.

Reviewers: mtomic, llugovic, mferencevic

Reviewed By: mtomic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1733
This commit is contained in:
Teon Banek 2018-11-14 13:06:26 +01:00
parent b77d186f58
commit a272fa2e6b
9 changed files with 313 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/lisp/CMakeLists.txt
View File

@ -5,6 +5,9 @@ set(lcp_src_files
${CMAKE_SOURCE_DIR}/src/lisp/lcp.asd
${CMAKE_SOURCE_DIR}/src/lisp/lcp-compile
${CMAKE_SOURCE_DIR}/src/lisp/package.lisp
${CMAKE_SOURCE_DIR}/src/lisp/types.lisp
${CMAKE_SOURCE_DIR}/src/lisp/code-gen.lisp
${CMAKE_SOURCE_DIR}/src/lisp/slk.lisp
${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp
${CMAKE_SOURCE_DIR}/src/lisp/lcp-test.lisp
${CMAKE_SOURCE_DIR}/tools/lcp)
@ -53,6 +56,9 @@ macro(define_add_lcp name main_src_files generated_lcp_files)
${CMAKE_SOURCE_DIR}/src/lisp/lcp.asd
${CMAKE_SOURCE_DIR}/src/lisp/lcp-compile
${CMAKE_SOURCE_DIR}/src/lisp/package.lisp
${CMAKE_SOURCE_DIR}/src/lisp/types.lisp
${CMAKE_SOURCE_DIR}/src/lisp/code-gen.lisp
${CMAKE_SOURCE_DIR}/src/lisp/slk.lisp
${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp
${CMAKE_SOURCE_DIR}/src/lisp/lcp-test.lisp
${CMAKE_SOURCE_DIR}/tools/lcp)

19
src/lisp/code-gen.lisp Normal file
View File

@ -0,0 +1,19 @@
;;;; This file contains common code for generating C++ code.
(in-package #:lcp)
(defun call-with-cpp-block-output (out fun &key semicolonp name)
"Surround the invocation of FUN by emitting '{' and '}' to OUT. If
SEMICOLONP is set, the closing '}' is suffixed with ';'. NAME is used to
prepend the starting block with a name, for example \"class MyClass\"."
(if name
(format out "~A {~%" name)
(write-line "{" out))
(funcall fun)
(if semicolonp (write-line "};" out) (write-line "}" out)))
(defmacro with-cpp-block-output ((out &rest rest &key semicolonp name) &body body)
"Surround BODY with emitting '{' and '}' to OUT. For additional arguments,
see `CALL-WITH-CPP-BLOCK-OUTPUT' documentation."
(declare (ignorable semicolonp name))
`(call-with-cpp-block-output ,out (lambda () ,@body) ,@rest))

144
src/lisp/lcp-test.lisp
View File

@ -137,3 +137,147 @@
(subtest "arrays"
(different-parse-test "char (*)[]" "char (*) []")
(different-parse-test "char (*)[4]" "char (*) [4]")))
(defun clang-format (cpp-string)
(with-input-from-string (s cpp-string)
(string-left-trim
'(#\Newline)
(uiop:run-program "clang-format -style=file" :input s :output '(:string :stripped t)))))
(defun is-generated (got expected)
(is (clang-format got) (clang-format expected) :test #'string=))
(defun undefine-cpp-types ()
(setf lcp::*cpp-classes* nil)
(setf lcp::*cpp-enums* nil))
(deftest "slk"
(subtest "function declarations"
(undefine-cpp-types)
(is-generated (lcp.slk:save-function-declaration-for-class
(lcp:define-struct test-struct ()
()))
"void Save(const TestStruct &self, slk::Builder *builder)")
(undefine-cpp-types)
(is-generated (lcp.slk:save-function-declaration-for-class
(lcp:define-class derived (base)
()))
"void Save(const Derived &self, slk::Builder *builder)")
(undefine-cpp-types)
(is-error (lcp.slk:save-function-declaration-for-class
(lcp:define-class derived (fst-base snd-base)
()))
'lcp.slk:slk-error)
(undefine-cpp-types)
(is-error (lcp.slk:save-function-declaration-for-class
(lcp:define-class (derived t-param) (base)
()))
'lcp.slk:slk-error)
(undefine-cpp-types)
(is-error (lcp.slk:save-function-declaration-for-class
(lcp:define-struct (test-struct fst-param snd-param) ()
()))
'lcp.slk:slk-error)
(undefine-cpp-types))
(subtest "save definitions"
(undefine-cpp-types)
(is-generated (lcp.slk:save-function-definition-for-class
(lcp:define-struct test-struct ()
((int-member :int64_t)
(vec-member "std::vector<SomeType>"))))
"void Save(const TestStruct &self, slk::Builder *builder) {
slk::Save(self.int_member, builder);
slk::Save(self.vec_member, builder);
}")
(undefine-cpp-types)
(is-generated (lcp.slk:save-function-definition-for-class
(lcp:define-struct test-struct ()
((skip-member :int64_t :dont-save t))))
"void Save(const TestStruct &self, slk::Builder *builder) {}")
(undefine-cpp-types)
(is-generated (lcp.slk:save-function-definition-for-class
(lcp:define-struct test-struct ()
((custom-member "SomeType"
:slk-save (lambda (member-name)
(check-type member-name string)
(format nil "self.~A.CustomSave(builder);" member-name))))))
"void Save(const TestStruct &self, slk::Builder *builder) {
self.custom_member.CustomSave(builder);
}")
(subtest "inheritance"
(undefine-cpp-types)
(is-error (lcp.slk:save-function-declaration-for-class
(lcp:define-struct derived (fst-base snd-base)
()))
'lcp.slk:slk-error)
(undefine-cpp-types)
(let ((base-class (lcp:define-struct base ()
((base-member :bool))))
(derived-class (lcp:define-struct derived (base)
((derived-member :int64_t)))))
(is-generated (lcp.slk:save-function-definition-for-class base-class)
"void Save(const Base &self, slk::Builder *builder) {
if (const auto &derived_derived = dynamic_cast<const Derived &>(self)) {
return slk::Save(derived_derived, builder);
}
slk::Save(self.base_member, builder);
}")
(is-generated (lcp.slk:save-function-definition-for-class derived-class)
"void Save(const Derived &self, slk::Builder *builder) {
// Save parent Base
{ slk::Save(self.base_member, builder); }
slk::Save(self.derived_member, builder);
}"))
(undefine-cpp-types)
(let ((abstract-base-class (lcp:define-struct abstract-base ()
((base-member :bool))
(:abstractp t)))
(derived-class (lcp:define-struct derived (abstract-base)
((derived-member :int64_t)))))
(is-generated (lcp.slk:save-function-definition-for-class abstract-base-class)
"void Save(const AbstractBase &self, slk::Builder *builder) {
if (const auto &derived_derived = dynamic_cast<const Derived &>(self)) {
return slk::Save(derived_derived, builder);
}
LOG(FATAL) << \"`AbstractBase` is marked as an abstract class!\";
}")
(is-generated (lcp.slk:save-function-definition-for-class derived-class)
"void Save(const Derived &self, slk::Builder *builder) {
// Save parent AbstractBase
{ slk::Save(self.base_member, builder); }
slk::Save(self.derived_member, builder);
}"))
(undefine-cpp-types)
(let ((base-templated-class (lcp:define-struct (base t-param) ()
((base-member :bool))))
(derived-class (lcp:define-struct derived (base)
((derived-member :int64_t)))))
(is-error (lcp.slk:save-function-definition-for-class base-templated-class)
'lcp.slk:slk-error)
(is-error (lcp.slk:save-function-definition-for-class derived-class)
'lcp.slk:slk-error))
(undefine-cpp-types)
(let ((base-class (lcp:define-struct base ()
((base-member :bool))))
(derived-templated-class (lcp:define-struct (derived t-param) (base)
((derived-member :int64_t)))))
(is-error (lcp.slk:save-function-definition-for-class base-class)
'lcp.slk:slk-error)
(is-error (lcp.slk:save-function-definition-for-class derived-templated-class)
'lcp.slk:slk-error)))
(subtest "non-public members"
(undefine-cpp-types)
(is-error (lcp.slk:save-function-definition-for-class
(lcp:define-class test-class ()
((public-member :bool :scope :public)
(private-member :int64_t))))
'lcp.slk:slk-error)
(undefine-cpp-types)
(is-error (lcp.slk:save-function-definition-for-class
(lcp:define-struct test-struct ()
((protected-member :int64_t :scope :protected)
(public-member :char))))
'lcp.slk:slk-error))))

2
src/lisp/lcp.asd
View File

@ -6,6 +6,8 @@
:serial t
:components ((:file "package")
(:file "types")
(:file "code-gen")
(:file "slk")
(:file "lcp"))
:in-order-to ((test-op (test-op "lcp/test"))))

16
src/lisp/lcp.lisp
View File

@ -34,22 +34,6 @@
(let ((cpp-name (cpp-variable-name (cpp-member-symbol cpp-member))))
(if struct cpp-name (format nil "~A_" cpp-name))))
(defun call-with-cpp-block-output (out fun &key semicolonp name)
"Surround the invocation of FUN by emitting '{' and '}' to OUT. If
SEMICOLONP is set, the closing '}' is suffixed with ';'. NAME is used to
prepend the starting block with a name, for example \"class MyClass\"."
(if name
(format out "~A {~%" name)
(write-line "{" out))
(funcall fun)
(if semicolonp (write-line "};" out) (write-line "}" out)))
(defmacro with-cpp-block-output ((out &rest rest &key semicolonp name) &body body)
"Surround BODY with emitting '{' and '}' to OUT. For additional arguments,
see `CALL-WITH-CPP-BLOCK-OUTPUT' documentation."
(declare (ignorable semicolonp name))
`(call-with-cpp-block-output ,out (lambda () ,@body) ,@rest))
(defun cpp-enum-definition (cpp-enum)
"Get C++ style `CPP-ENUM' definition as a string."
(declare (type cpp-enum cpp-enum))

6
src/lisp/package.lisp
View File

@ -18,3 +18,9 @@
#:capnp-save-enum
#:capnp-load-enum
#:process-file))
(defpackage #:lcp.slk
(:use #:cl)
(:export #:save-function-declaration-for-class
#:save-function-definition-for-class
#:slk-error))

123
src/lisp/slk.lisp Normal file
View File

@ -0,0 +1,123 @@
;;;; This file contains code generation for serialization to our Save Load
;;;; Kit (SLK). It works very similarly to Cap'n Proto serialization, but
;;;; without the schema generation.
(in-package #:lcp.slk)
(define-condition slk-error (error)
((message :type string :initarg :message :reader slk-error-message))
(:report (lambda (condition stream)
(write-string (slk-error-message condition) stream))))
(defun save-function-declaration-for-class (cpp-class)
"Generate SLK save function declaration for CPP-CLASS. Note that the code
generation expects the declarations and definitions to be in `slk` namespace."
(check-type cpp-class lcp::cpp-class)
(when (lcp::cpp-type-type-params cpp-class)
(error 'slk-error :message
(format nil "Don't know how to save templated class '~A'"
(lcp::cpp-type-base-name cpp-class))))
(when (< 1 (list-length (lcp::cpp-class-super-classes cpp-class)))
(error 'slk-error :message
(format nil "Don't know how to save multiple parents of '~A'"
(lcp::cpp-type-base-name cpp-class))))
(let ((self-arg
(list 'self (format nil "const ~A &"
(lcp::cpp-type-decl cpp-class :namespace nil))))
(builder-arg (list 'builder "slk::Builder *")))
(lcp::cpp-function-declaration
"Save" :args (list self-arg builder-arg)
:type-params (lcp::cpp-type-type-params cpp-class))))
(defun save-members (cpp-class)
"Generate code for saving members of CPP-CLASS. Raise `SLK-ERROR' if the
serializable member has no public access."
(with-output-to-string (s)
(dolist (member (lcp::cpp-class-members-for-save cpp-class))
(let ((member-name (lcp::cpp-member-name member :struct (lcp::cpp-class-structp cpp-class))))
(when (not (eq :public (lcp::cpp-member-scope member)))
(error 'slk-error :message
(format nil "Cannot save non-public member '~A' of '~A'"
(lcp::cpp-member-symbol member) (lcp::cpp-type-base-name cpp-class))))
(cond
((lcp::cpp-member-slk-save member)
;; Custom save function
(write-line (lcp::cpp-code (funcall (lcp::cpp-member-slk-save member)
member-name))
s))
;; TODO: Extra args for cpp-class members
(t
(format s "slk::Save(self.~A, builder);~%" member-name)))))))
(defun save-parents-recursively (cpp-class)
"Generate code for saving members of all parents, recursively. Raise
`SLK-ERROR' if trying to save templated parent class or if using multiple
inheritance."
(when (< 1 (list-length (lcp::cpp-class-super-classes cpp-class)))
(error 'slk-error :message
(format nil "Don't know how to save multiple parents of '~A'"
(lcp::cpp-type-base-name cpp-class))))
(with-output-to-string (s)
;; TODO: Stop recursing to parents if CPP-CLASS is marked as base for
;; serialization.
(dolist (parent (lcp::cpp-class-super-classes cpp-class))
(let ((parent-class (lcp::find-cpp-class parent)))
(assert parent-class)
(when (lcp::cpp-type-type-params parent-class)
(error 'slk-error :message
(format nil "Don't know how to save templated parent class '~A'"
(lcp::cpp-type-base-name parent-class))))
(format s "// Save parent ~A~%" (lcp::cpp-type-name parent))
(lcp::with-cpp-block-output (s)
(write-string (save-parents-recursively parent-class) s)
(write-string (save-members parent-class) s))))))
(defun forward-save-to-subclasses (cpp-class)
"Generate code which forwards the serialization to derived classes of
CPP-CLASS. Raise `SLK-ERROR' if a derived class has template parameters."
(with-output-to-string (s)
(let ((subclasses (lcp::direct-subclasses-of cpp-class)))
(dolist (subclass subclasses)
(when (lcp::cpp-type-type-params subclass)
(error 'slk-error :message
(format nil "Don't know how to save derived templated class '~A'"
(lcp::cpp-type-base-name subclass))))
(let ((derived-class (lcp::cpp-type-name subclass))
(derived-var (lcp::cpp-variable-name (lcp::cpp-type-base-name subclass)))
;; TODO: Extra save arguments
(extra-args nil))
(format s "if (const auto &~A_derived = dynamic_cast<const ~A &>(self)) {
return slk::Save(~A_derived, builder~{, ~A~}); }~%"
derived-var derived-class derived-var extra-args))))))
(defun save-function-code-for-class (cpp-class)
"Generate code for serializing CPP-CLASS. Raise `SLK-ERROR' on unsupported
C++ constructs, mostly related to templates."
(when (lcp::cpp-type-type-params cpp-class)
(error 'slk-error :message
(format nil "Don't know how to save templated class '~A'"
(lcp::cpp-type-base-name cpp-class))))
(with-output-to-string (s)
(cond
((lcp::direct-subclasses-of cpp-class)
(write-string (forward-save-to-subclasses cpp-class) s)
(if (lcp::cpp-class-abstractp cpp-class)
(format s "LOG(FATAL) << \"`~A` is marked as an abstract class!\";"
(lcp::cpp-type-name cpp-class))
(progn
(write-string (save-parents-recursively cpp-class) s)
(write-string (save-members cpp-class) s))))
(t
;; TODO: Write some sort of type ID for derived classes
(write-string (save-parents-recursively cpp-class) s)
(write-string (save-members cpp-class) s)))))
(defun save-function-definition-for-class (cpp-class)
"Generate SLK save function. Raise `SLK-ERROR' if an unsupported or invalid
class definition is encountered during code generation. Note that the code
generation expects the declarations and definitions to be in `slk` namespace."
(check-type cpp-class lcp::cpp-class)
(with-output-to-string (cpp-out)
(lcp::with-cpp-block-output
(cpp-out :name (save-function-declaration-for-class cpp-class))
(write-line (save-function-code-for-class cpp-class) cpp-out))))

13
src/lisp/types.lisp
View File

@ -125,6 +125,9 @@
;; TODO: Support giving a name for reader function.
(reader nil :type boolean :read-only t)
(documentation nil :type (or null string) :read-only t)
;; If T, skips this member in serialization code generation. The member may
;; still be deserialized with custom load hook.
(dont-save nil :type boolean :read-only t)
;; CAPNP-TYPE may be a string specifying the type, or a list of
;; (member-symbol "capnp-type") specifying a union type.
(capnp-type nil :type (or null string list) :read-only t)
@ -132,7 +135,11 @@
;; Custom saving and loading code. May be a function which takes 2
;; args: (builder-or-reader member-name) and needs to return C++ code.
(capnp-save nil :type (or null function (eql :dont-save)) :read-only t)
(capnp-load nil :type (or null function) :read-only t))
(capnp-load nil :type (or null function) :read-only t)
;; May be a function which takes 1 argument, member-name. It needs to
;; return C++ code.
(slk-save nil :type (or null function) :read-only t)
(slk-load nil :type (or null function) :read-only t))
(defstruct capnp-opts
"Cap'n Proto serialization options for C++ class."
@ -173,6 +180,10 @@
(defvar *cpp-classes* nil "List of defined classes from LCP file")
(defvar *cpp-enums* nil "List of defined enums from LCP file")
(defun cpp-class-members-for-save (cpp-class)
(check-type cpp-class cpp-class)
(remove-if #'cpp-member-dont-save (cpp-class-members cpp-class)))
(defun make-cpp-primitive-type (name)
"Create an instance of CPP-PRIMITIVE-TYPE given the arguments."
(check-type name cpp-primitive-type-keywords)

2
src/query/plan/operator.cpp
View File

@ -627,7 +627,7 @@ auto ExpandFromVertex(const VertexAccessor &vertex,
[direction](const EdgeAccessor &edge) {
return std::make_pair(edge, direction);
},
std::move(vertices));
std::forward<decltype(vertices)>(vertices));
};
// prepare a vector of elements we'll pass to the itertools