tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generate TypeInfo object for all LCP defined classes

Browse Source 
Summary:
TypeInfo will be needed for upcoming serialization via SLK. This diff
changes the already defined TypeInfo by removing the reliance on
capnp::typeId calls. The struct itself is now in utils.

Hopefully, this shouldn't break our RPC stack due to new ID generation.

Reviewers: mferencevic, mtomic, llugovic

Reviewed By: mtomic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1735
This commit is contained in:
Teon Banek 2018-11-16 10:55:37 +01:00
parent a272fa2e6b
commit 8ed1fbbbe1
16 changed files with 156 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/communication/rpc/client.hpp
View File

@ -50,7 +50,7 @@ class Client {
load,
Args &&... args) {
typename TRequestResponse::Request request(std::forward<Args>(args)...);
auto req_type = TRequestResponse::Request::TypeInfo;
auto req_type = TRequestResponse::Request::kType;
VLOG(12) << "[RpcClient] sent " << req_type.name;
::capnp::MallocMessageBuilder req_msg;
{
@ -64,7 +64,7 @@ class Client {
}
auto response = Send(&req_msg);
auto res_msg = response.getRoot<capnp::Message>();
auto res_type = TRequestResponse::Response::TypeInfo;
auto res_type = TRequestResponse::Response::kType;
if (res_msg.getTypeId() != res_type.id) {
// Since message_id was checked in private Call function, this means
// something is very wrong (probably on the server side).

32
src/communication/rpc/messages.hpp
View File

@ -3,42 +3,16 @@
#include <cstdint>
#include <memory>
#include "utils/typeinfo.hpp"
namespace communication::rpc {
using MessageSize = uint32_t;
/// Type information on a RPC message.
/// Each message should have a static member `TypeInfo` with this information.
struct MessageType {
/// Unique ID for a message.
uint64_t id;
/// Pretty name of the type.
std::string name;
};
inline bool operator==(const MessageType &a, const MessageType &b) {
return a.id == b.id;
}
inline bool operator!=(const MessageType &a, const MessageType &b) {
return a.id != b.id;
}
inline bool operator<(const MessageType &a, const MessageType &b) {
return a.id < b.id;
}
inline bool operator<=(const MessageType &a, const MessageType &b) {
return a.id <= b.id;
}
inline bool operator>(const MessageType &a, const MessageType &b) {
return a.id > b.id;
}
inline bool operator>=(const MessageType &a, const MessageType &b) {
return a.id >= b.id;
}
/// Each RPC is defined via this struct.
///
/// `TRequest` and `TResponse` are required to be classes which have a static
/// member `TypeInfo` of `MessageType` type. This is used for proper
/// member `kType` of `utils::TypeInfo` type. This is used for proper
/// registration and deserialization of RPC types. Additionally, both `TRequest`
/// and `TResponse` are required to define a nested `Capnp` type, which
/// corresponds to the Cap'n Proto schema type, as well as defined the following

28
src/communication/rpc/server.hpp
View File

@ -38,13 +38,13 @@ class Server {
std::lock_guard<std::mutex> guard(lock_);
CHECK(!server_.IsRunning()) << "You can't register RPCs when the server is running!";
RpcCallback rpc;
rpc.req_type = TRequestResponse::Request::TypeInfo;
rpc.res_type = TRequestResponse::Response::TypeInfo;
rpc.req_type = TRequestResponse::Request::kType;
rpc.res_type = TRequestResponse::Response::kType;
rpc.callback = [callback = callback](const auto &reader, auto *builder) {
auto req_data =
reader.getData()
.template getAs<typename TRequestResponse::Request::Capnp>();
builder->setTypeId(TRequestResponse::Response::TypeInfo.id);
builder->setTypeId(TRequestResponse::Response::kType.id);
auto data_builder = builder->initData();
auto res_builder =
data_builder
@ -52,12 +52,12 @@ class Server {
callback(req_data, &res_builder);
};
if (extended_callbacks_.find(TRequestResponse::Request::TypeInfo.id) !=
if (extended_callbacks_.find(TRequestResponse::Request::kType.id) !=
extended_callbacks_.end()) {
LOG(FATAL) << "Callback for that message type already registered!";
}
auto got = callbacks_.insert({TRequestResponse::Request::TypeInfo.id, rpc});
auto got = callbacks_.insert({TRequestResponse::Request::kType.id, rpc});
CHECK(got.second) << "Callback for that message type already registered";
VLOG(12) << "[RpcServer] register " << rpc.req_type.name << " -> "
<< rpc.res_type.name;
@ -72,14 +72,14 @@ class Server {
std::lock_guard<std::mutex> guard(lock_);
CHECK(!server_.IsRunning()) << "You can't register RPCs when the server is running!";
RpcExtendedCallback rpc;
rpc.req_type = TRequestResponse::Request::TypeInfo;
rpc.res_type = TRequestResponse::Response::TypeInfo;
rpc.req_type = TRequestResponse::Request::kType;
rpc.res_type = TRequestResponse::Response::kType;
rpc.callback = [callback = callback](const io::network::Endpoint &endpoint,
const auto &reader, auto *builder) {
auto req_data =
reader.getData()
.template getAs<typename TRequestResponse::Request::Capnp>();
builder->setTypeId(TRequestResponse::Response::TypeInfo.id);
builder->setTypeId(TRequestResponse::Response::kType.id);
auto data_builder = builder->initData();
auto res_builder =
data_builder
@ -87,13 +87,13 @@ class Server {
callback(endpoint, req_data, &res_builder);
};
if (callbacks_.find(TRequestResponse::Request::TypeInfo.id) !=
if (callbacks_.find(TRequestResponse::Request::kType.id) !=
callbacks_.end()) {
LOG(FATAL) << "Callback for that message type already registered!";
}
auto got =
extended_callbacks_.insert({TRequestResponse::Request::TypeInfo.id, rpc});
extended_callbacks_.insert({TRequestResponse::Request::kType.id, rpc});
CHECK(got.second) << "Callback for that message type already registered";
VLOG(12) << "[RpcServer] register " << rpc.req_type.name << " -> "
<< rpc.res_type.name;
@ -103,20 +103,20 @@ class Server {
friend class Session;
struct RpcCallback {
MessageType req_type;
utils::TypeInfo req_type;
std::function<void(const capnp::Message::Reader &,
capnp::Message::Builder *)>
callback;
MessageType res_type;
utils::TypeInfo res_type;
};
struct RpcExtendedCallback {
MessageType req_type;
utils::TypeInfo req_type;
std::function<void(const io::network::Endpoint &,
const capnp::Message::Reader &,
capnp::Message::Builder *)>
callback;
MessageType res_type;
utils::TypeInfo res_type;
};
std::mutex lock_;

2
src/distributed/bfs_rpc_messages.lcp
View File

@ -165,7 +165,6 @@ cpp<#
(:public
#>cpp
using Capnp = capnp::ReconstructPathReq;
static const communication::rpc::MessageType TypeInfo;
ReconstructPathReq() {}
@ -204,7 +203,6 @@ cpp<#
(:public
#>cpp
using Capnp = capnp::ReconstructPathRes;
static const communication::rpc::MessageType TypeInfo;
ReconstructPathRes() {}

1
src/durability/distributed/state_delta.lcp
View File

@ -9,6 +9,7 @@
#include "storage/common/types/types.hpp"
#include "storage/distributed/address_types.hpp"
#include "storage/distributed/gid.hpp"
#include "utils/typeinfo.hpp"
cpp<#
(lcp:namespace database)

1
src/durability/single_node/state_delta.lcp
View File

@ -9,6 +9,7 @@
#include "storage/common/types/property_value.hpp"
#include "storage/common/types/types.hpp"
#include "storage/single_node/gid.hpp"
#include "utils/typeinfo.hpp"
class Vertex;
class Edge;

1
src/durability/single_node_ha/state_delta.lcp
View File

@ -9,6 +9,7 @@
#include "storage/common/types/property_value.hpp"
#include "storage/common/types/types.hpp"
#include "storage/single_node_ha/gid.hpp"
#include "utils/typeinfo.hpp"
class Vertex;
class Edge;

15
src/lisp/CMakeLists.txt
View File

@ -26,6 +26,10 @@ add_custom_target(lcp
#
# The `add_lcp` function expects at least a single argument, path to lcp file.
# Each added file is standalone and we avoid recompiling everything.
#
# By default, each `.lcp` file will produce a `.hpp` and `.cpp` file. To tell
# CMake that no `.cpp` file will be generated, pass a NO_CPP option.
#
# You may pass a CAPNP_SCHEMA <id> keyword argument to generate the Cap'n Proto
# serialization code from .lcp file. You still need to add the generated capnp
# file through `add_capnp` function. To generate the <id> use `capnp id`
@ -34,21 +38,24 @@ add_custom_target(lcp
# information will break serialization between different compilations.
macro(define_add_lcp name main_src_files generated_lcp_files)
function(${name} lcp_file)
set(options NO_CPP)
set(one_value_kwargs CAPNP_SCHEMA)
set(multi_value_kwargs DEPENDS)
# NOTE: ${${}ARGN} syntax escapes evaluating macro's ARGN variable; see:
# https://stackoverflow.com/questions/50365544/how-to-access-enclosing-functions-arguments-from-within-a-macro
cmake_parse_arguments(KW "" "${one_value_kwargs}" "${multi_value_kwargs}" ${${}ARGN})
cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${${}ARGN})
string(REGEX REPLACE "\.lcp$" ".hpp" h_file
"${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}")
if (NOT KW_NO_CPP)
set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}.cpp)
# Update *global* main_src_files
set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
endif()
if (KW_CAPNP_SCHEMA)
string(REGEX REPLACE "\.lcp$" ".capnp" capnp_file
"${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}")
set(capnp_id ${KW_CAPNP_SCHEMA})
set(capnp_depend capnproto-proj)
set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}.cpp)
# Update *global* main_src_files
set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
endif()
# Repeat the `lcp_src_files` because this is a macro and the variable is
# not visible when invoked in another file.

7
src/lisp/lcp-test.lisp
View File

@ -138,6 +138,13 @@
(different-parse-test "char (*)[]" "char (*) []")
(different-parse-test "char (*)[4]" "char (*) [4]")))
(deftest "fnv-hash"
(subtest "fnv1a64"
(is (lcp::fnv1a64-hash-string "query::plan::LogicalOperator")
#xCF6E3316FE845113)
(is (lcp::fnv1a64-hash-string "SomeString") #x1730D3E779304E6C)
(is (lcp::fnv1a64-hash-string "SomeStrink") #x1730D7E779305538)))
(defun clang-format (cpp-string)
(with-input-from-string (s cpp-string)
(string-left-trim

89
src/lisp/lcp.lisp
View File

@ -6,9 +6,25 @@
(defvar +vim-read-only+ "vim: readonly")
(defvar +emacs-read-only+ "-*- buffer-read-only: t; -*-")
(defvar *generating-cpp-impl-p* nil
"T if we are currently writing the .cpp file.")
(eval-when (:compile-toplevel :load-toplevel :execute)
(set-dispatch-macro-character #\# #\> #'|#>-reader|))
(defun fnv1a64-hash-string (string)
"Produce (UNSIGNED-BYTE 64) hash of the given STRING using FNV-1a algorithm.
See https://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash."
(check-type string string)
(let ((hash 14695981039346656037) ;; offset basis
(prime 1099511628211))
(declare (type (unsigned-byte 64) hash prime))
(loop for c across string do
(setf hash (mod (* (boole boole-xor hash (char-code c)) prime)
(expt 2 64) ;; Fit to 64bit
)))
hash))
(defun cpp-documentation (documentation)
"Convert DOCUMENTATION to Doxygen style string."
(declare (type string documentation))
@ -94,10 +110,15 @@ NIL, returns a string."
(with-cpp-block-output (s :semicolonp t)
(let ((reader-members (remove-if (complement #'cpp-member-reader)
(cpp-class-members cpp-class))))
(when (or (cpp-class-public cpp-class) (cpp-class-members-scoped :public) reader-members)
(when (or (cpp-class-public cpp-class) (cpp-class-members-scoped :public) reader-members
;; We at least have public TypeInfo object for non-template classes.
(not (cpp-type-type-params cpp-class)))
(unless (cpp-class-structp cpp-class)
(write-line " public:" s))
(format s "~{~A~%~}" (mapcar #'cpp-code (cpp-class-public cpp-class)))
(unless (cpp-type-type-params cpp-class)
;; Skip generating TypeInfo for template classes.
(write-line "static const utils::TypeInfo kType;" s))
(format s "~%~{~A~%~}" (mapcar #'cpp-code (cpp-class-public cpp-class)))
(format s "~{~%~A~}~%" (mapcar #'cpp-member-reader-definition reader-members))
(format s "~{ ~%~A~}~%"
(mapcar #'member-declaration (cpp-class-members-scoped :public)))))
@ -110,7 +131,23 @@ NIL, returns a string."
(write-line " private:" s)
(format s "~{~A~%~}" (mapcar #'cpp-code (cpp-class-private cpp-class)))
(format s "~{ ~%~A~}~%"
(mapcar #'member-declaration (cpp-class-members-scoped :private))))))))
(mapcar #'member-declaration (cpp-class-members-scoped :private)))))
;; Define the TypeInfo object. Relies on the fact that *CPP-IMPL* is
;; processed later.
(unless (cpp-type-type-params cpp-class)
(let ((typeinfo-def
(format nil "const utils::TypeInfo ~A::kType{0x~XULL, \"~a\"};~%"
(if *generating-cpp-impl-p*
(cpp-type-name cpp-class)
;; Use full type declaration if class definition
;; isn't inside the .cpp file.
(cpp-type-decl cpp-class))
;; Use full type declaration for hash
(fnv1a64-hash-string (cpp-type-decl cpp-class))
(cpp-type-name cpp-class))))
(if *generating-cpp-impl-p*
(write-line typeinfo-def s)
(in-impl typeinfo-def)))))))
(defun cpp-function-declaration (name &key args (returns "void") type-params)
"Generate a C++ top level function declaration named NAME as a string. ARGS
@ -1285,12 +1322,6 @@ enums which aren't defined in LCP."
(flet ((decl-type-info (class-name)
#>cpp
using Capnp = capnp::${class-name};
static const communication::rpc::MessageType TypeInfo;
cpp<#)
(def-type-info (class-name)
#>cpp
const communication::rpc::MessageType
${class-name}::TypeInfo{::capnp::typeId<${class-name}::Capnp>(), "${class-name}"};
cpp<#)
(def-constructor (class-name members)
(let ((full-constructor
@ -1335,14 +1366,12 @@ enums which aren't defined in LCP."
,(decl-type-info req-name)
,(def-constructor req-name (second request)))
(:serialize :capnp :base t))
(in-impl ,(def-type-info req-name))
(define-struct ,res-sym ()
,@(cdr response)
(:public
,(decl-type-info res-name)
,(def-constructor res-name (second response)))
(:serialize :capnp :base t))
(in-impl ,(def-type-info res-name))
,rpc-decl))))
(defun read-lcp (filepath)
@ -1378,6 +1407,11 @@ namespaces."
(declare (type (function (function)) fun))
(let (open-namespaces)
(funcall fun (lambda (namespaces)
;; No namespaces is global namespace
(unless namespaces
(dolist (to-close open-namespaces)
(declare (ignore to-close))
(format out "~%}")))
;; Check if we need to open or close namespaces
(loop for namespace in namespaces
with unmatched = open-namespaces do
@ -1498,19 +1532,22 @@ file."
(cpp-enum
(format out "~A;~%" (cpp-enum-to-capnp-function-declaration type-for-capnp))
(format out "~A;~%" (cpp-enum-from-capnp-function-declaration type-for-capnp)))))))
;; When we have either capnp or C++ code for the .cpp file, generate the .cpp file
;; When we have either capnp or C++ code for the .cpp file, generate
;; the .cpp file. Note, that some code may rely on the fact that .cpp
;; file is generated after .hpp.
(when (or *cpp-impl* types-for-capnp)
(with-open-file (out cpp-file :direction :output :if-exists :supersede)
(format out "~@{// ~A~%~}" +emacs-read-only+ +vim-read-only+)
(format out "// DO NOT EDIT! Generated using LCP from '~A'~2%"
(file-namestring lcp-file))
(format out "#include \"~A\"~2%" (file-namestring hpp-file))
;; First output the C++ code from the user
(with-namespaced-output (out open-namespace)
(dolist (cpp *cpp-impl*)
(destructuring-bind (namespaces . code) cpp
(open-namespace namespaces)
(write-line (cpp-code code) out))))
(when types-for-capnp
(generate-capnp types-for-capnp :capnp-file capnp-file :capnp-id capnp-id
:cpp-out out :lcp-file lcp-file))))))))
(let ((*generating-cpp-impl-p* t))
(with-open-file (out cpp-file :direction :output :if-exists :supersede)
(format out "~@{// ~A~%~}" +emacs-read-only+ +vim-read-only+)
(format out "// DO NOT EDIT! Generated using LCP from '~A'~2%"
(file-namestring lcp-file))
(format out "#include \"~A\"~2%" (file-namestring hpp-file))
;; First output the C++ code from the user
(with-namespaced-output (out open-namespace)
(dolist (cpp *cpp-impl*)
(destructuring-bind (namespaces . code) cpp
(open-namespace namespaces)
(write-line (cpp-code code) out))))
(when types-for-capnp
(generate-capnp types-for-capnp :capnp-file capnp-file :capnp-id capnp-id
:cpp-out out :lcp-file lcp-file)))))))))

1
src/query/frontend/ast/ast.lcp
View File

@ -11,6 +11,7 @@
#include "query/typed_value.hpp"
#include "storage/common/types/property_value.hpp"
#include "storage/common/types/types.hpp"
#include "utils/typeinfo.hpp"
// Hash function for the key in pattern atom property maps.
namespace std {

37
src/utils/typeinfo.hpp Normal file
View File

@ -0,0 +1,37 @@
#pragma once
namespace utils {
/// Type information on a C++ type.
///
/// You should embed this structure as a static constant member `kType` and make
/// sure you generate a unique ID for it. Also, if your type has inheritance,
/// you may want to add a `virtual utils::TypeInfo GetType();` method to get the
/// runtime type.
struct TypeInfo {
/// Unique ID for the type.
uint64_t id;
/// Pretty name of the type.
std::string name;
};
inline bool operator==(const TypeInfo &a, const TypeInfo &b) {
return a.id == b.id;
}
inline bool operator!=(const TypeInfo &a, const TypeInfo &b) {
return a.id != b.id;
}
inline bool operator<(const TypeInfo &a, const TypeInfo &b) {
return a.id < b.id;
}
inline bool operator<=(const TypeInfo &a, const TypeInfo &b) {
return a.id <= b.id;
}
inline bool operator>(const TypeInfo &a, const TypeInfo &b) {
return a.id > b.id;
}
inline bool operator>=(const TypeInfo &a, const TypeInfo &b) {
return a.id >= b.id;
}
} // namespace utils

4
tests/benchmark/rpc.cpp
View File

@ -13,7 +13,7 @@
struct EchoMessage {
using Capnp = ::capnp::AnyPointer;
static const communication::rpc::MessageType TypeInfo;
static const utils::TypeInfo kType;
EchoMessage() {} // Needed for serialization.
EchoMessage(const std::string &data) : data(data) {}
@ -31,7 +31,7 @@ void Load(EchoMessage *echo, const ::capnp::AnyPointer::Reader &reader) {
echo->data = list_reader[0];
}
const communication::rpc::MessageType EchoMessage::TypeInfo{2, "EchoMessage"};
const utils::TypeInfo EchoMessage::kType{2, "EchoMessage"};
using Echo = communication::rpc::RequestResponse<EchoMessage, EchoMessage>;

12
tests/unit/rpc.cpp
View File

@ -15,7 +15,7 @@ using namespace std::literals::chrono_literals;
struct SumReq {
using Capnp = ::capnp::AnyPointer;
static const MessageType TypeInfo;
static const utils::TypeInfo kType;
SumReq() {} // Needed for serialization.
SumReq(int x, int y) : x(x), y(y) {}
@ -35,11 +35,11 @@ void Load(SumReq *sum, const ::capnp::AnyPointer::Reader &reader) {
sum->y = list_reader[1];
}
const MessageType SumReq::TypeInfo{0, "SumReq"};
const utils::TypeInfo SumReq::kType{0, "SumReq"};
struct SumRes {
using Capnp = ::capnp::AnyPointer;
static const MessageType TypeInfo;
static const utils::TypeInfo kType;
SumRes() {} // Needed for serialization.
SumRes(int sum) : sum(sum) {}
@ -57,13 +57,13 @@ void Load(SumRes *res, const ::capnp::AnyPointer::Reader &reader) {
res->sum = list_reader[0];
}
const MessageType SumRes::TypeInfo{1, "SumRes"};
const utils::TypeInfo SumRes::kType{1, "SumRes"};
using Sum = RequestResponse<SumReq, SumRes>;
struct EchoMessage {
using Capnp = ::capnp::AnyPointer;
static const MessageType TypeInfo;
static const utils::TypeInfo kType;
EchoMessage() {} // Needed for serialization.
EchoMessage(const std::string &data) : data(data) {}
@ -81,7 +81,7 @@ void Load(EchoMessage *echo, const ::capnp::AnyPointer::Reader &reader) {
echo->data = list_reader[0];
}
const MessageType EchoMessage::TypeInfo{2, "EchoMessage"};
const utils::TypeInfo EchoMessage::kType{2, "EchoMessage"};
using Echo = RequestResponse<EchoMessage, EchoMessage>;

6
tools/analyze_rpc_calls
View File

@ -37,6 +37,12 @@ def parse_capnp_header(fname):
return ret
# TODO(mferencevic): Update this to parse .cpp files (99% are .lcp.cpp),
# containing the following line.
#
# const utils::TypeInfo <class-name>::kType{<id-hex>, "<class-name>"};
#
# Note that clang-format may break the line at any of the spaces or after '{'.
def parse_all_capnp_headers(dirname):
ids = {}
ret = subprocess.run(["find", dirname, "-name", "*.capnp.h"],

2
tools/lcp
View File

@ -36,6 +36,6 @@ filename=`basename $lcp_file .lcp`
hpp_file="$(dirname $lcp_file)/$filename.hpp"
clang-format -style=file -i $hpp_file
if [[ $# -eq 2 && -w "$lcp_file.cpp" ]]; then
if [[ -w "$lcp_file.cpp" ]]; then
clang-format -style=file -i "$lcp_file.cpp"
fi