diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 59ac57995..36ef581e6 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -164,9 +164,3 @@ import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR}) # Setup json import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR}) - -# Setup cereal -import_header_library(cereal "${CMAKE_CURRENT_SOURCE_DIR}/cereal/include") -# Make cereal multithreaded by passing -DCEREAL_THREAD_SAFE=1 (note that -D is omitted below). -set_property(TARGET cereal PROPERTY - INTERFACE_COMPILE_DEFINITIONS CEREAL_THREAD_SAFE=1) diff --git a/libs/setup.sh b/libs/setup.sh index 453b44808..b53b4d8a1 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -123,7 +123,3 @@ cd .. # git clone https://github.com/r-lyeh/ltalloc.git ltalloc_tag="43b51c14857111f993f277c46151fdfac91525a2" # Nov 16, 2017 clone git://deps.memgraph.io/ltalloc.git ltalloc $ltalloc_tag - -# cereal -#git clone https://github.com/USCiLab/cereal.git -clone git://deps.memgraph.io/cereal.git cereal v1.2.2 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4944caec2..95b4ccc57 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,8 +10,8 @@ set(memgraph_src_files communication/rpc/rpc.cpp data_structures/concurrent/skiplist_gc.cpp database/graph_db.cpp - database/graph_db_config.cpp database/graph_db_accessor.cpp + database/graph_db_config.cpp database/state_delta.cpp distributed/coordination_master.cpp distributed/coordination_worker.cpp @@ -50,7 +50,7 @@ set(memgraph_src_files # ----------------------------------------------------------------------------- # memgraph_lib depend on these libraries -set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools cereal +set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools antlr_opencypher_parser_lib dl glog gflags ${Boost_SERIALIZATION_LIBRARY_RELEASE}) if (USE_LTALLOC) @@ -124,4 +124,3 @@ set(examples ${CMAKE_SOURCE_DIR}/release/examples) install(CODE "execute_process(COMMAND ${examples}/build_examples WORKING_DIRECTORY ${examples})") install(DIRECTORY ${examples}/build/ DESTINATION share/memgraph/examples) - diff --git a/src/communication/messaging/distributed.hpp b/src/communication/messaging/distributed.hpp index c185a5e46..c838450c6 100644 --- a/src/communication/messaging/distributed.hpp +++ b/src/communication/messaging/distributed.hpp @@ -18,14 +18,6 @@ #include "data_structures/queue.hpp" #include "protocol.hpp" -#include "cereal/archives/binary.hpp" -#include "cereal/types/base_class.hpp" -#include "cereal/types/memory.hpp" -#include "cereal/types/polymorphic.hpp" -#include "cereal/types/string.hpp" -#include "cereal/types/utility.hpp" -#include "cereal/types/vector.hpp" - #include "communication/server.hpp" #include "io/network/network_endpoint.hpp" #include "threading/sync/spinlock.hpp" diff --git a/src/communication/messaging/local.cpp b/src/communication/messaging/local.cpp index 38d3b2e5b..226855407 100644 --- a/src/communication/messaging/local.cpp +++ b/src/communication/messaging/local.cpp @@ -58,4 +58,5 @@ std::unique_ptr EventStream::Await( }; void EventStream::Shutdown() { queue_.Shutdown(); } -} + +} // namespace communication::messaging diff --git a/src/communication/messaging/local.hpp b/src/communication/messaging/local.hpp index 66cd21e60..8a80d34a5 100644 --- a/src/communication/messaging/local.hpp +++ b/src/communication/messaging/local.hpp @@ -4,8 +4,9 @@ #include #include #include +#include -#include "cereal/types/memory.hpp" +#include "boost/serialization/access.hpp" #include "data_structures/queue.hpp" @@ -18,9 +19,6 @@ class Message { public: virtual ~Message() {} - template - void serialize(Archive &) {} - /** * Run-time type identification that is used for callbacks. * @@ -28,6 +26,12 @@ class Message { * this class */ std::type_index type_index() const { return typeid(*this); } + + private: + friend boost::serialization::access; + + template + void serialize(TArchive &, unsigned int) {} }; class EventStream; @@ -106,4 +110,5 @@ class EventStream { std::string name_; Queue> queue_; }; + } // namespace communication::messaging diff --git a/src/communication/messaging/protocol.cpp b/src/communication/messaging/protocol.cpp index e31ee0701..86720783e 100644 --- a/src/communication/messaging/protocol.cpp +++ b/src/communication/messaging/protocol.cpp @@ -1,11 +1,15 @@ #include +#include "boost/archive/binary_iarchive.hpp" +#include "boost/archive/binary_oarchive.hpp" +#include "boost/serialization/unique_ptr.hpp" +#include "fmt/format.h" +#include "glog/logging.h" + #include "communication/messaging/distributed.hpp" #include "communication/messaging/local.hpp" #include "communication/messaging/protocol.hpp" - -#include "fmt/format.h" -#include "glog/logging.h" +#include "communication/rpc/messages-inl.hpp" namespace communication::messaging { @@ -33,11 +37,11 @@ void Session::Execute() { buffer_.Shift(sizeof(SizeT)); // TODO: check for exceptions - std::istringstream stream; + std::stringstream stream; stream.str(std::string(reinterpret_cast(buffer_.data()), len_data)); - ::cereal::BinaryInputArchive iarchive{stream}; + boost::archive::binary_iarchive archive(stream); std::unique_ptr message{nullptr}; - iarchive(message); + archive >> message; buffer_.Shift(len_data); LocalWriter writer(system_, channel); @@ -86,9 +90,9 @@ void SendMessage(const std::string &address, uint16_t port, } // Serialize and send message - std::ostringstream stream; - ::cereal::BinaryOutputArchive oarchive(stream); - oarchive(message); + std::stringstream stream; + boost::archive::binary_oarchive archive(stream); + archive << message; const std::string &buffer = stream.str(); int64_t message_size = 2 * sizeof(SizeT) + buffer.size() + channel.size(); diff --git a/src/communication/raft/network_common.hpp b/src/communication/raft/network_common.hpp index 2913dfe1e..d1de71c41 100644 --- a/src/communication/raft/network_common.hpp +++ b/src/communication/raft/network_common.hpp @@ -1,6 +1,7 @@ #pragma once -#include "cereal/cereal.hpp" +#include "boost/serialization/access.hpp" +#include "boost/serialization/base_object.hpp" #include "communication/messaging/distributed.hpp" #include "communication/raft/raft.hpp" @@ -15,10 +16,15 @@ struct PeerRpcRequest : public messaging::Message { RequestVoteRequest request_vote; AppendEntriesRequest append_entries; - template - void serialize(Archive &ar) { - ar(cereal::virtual_base_class(this), type, request_vote, - append_entries); + private: + friend class boost::serialization::access; + + template + void serialize(TArchive &ar, unsigned int) { + ar &boost::serialization::base_object(*this); + ar &type; + ar &request_vote; + ar &append_entries; } }; @@ -27,10 +33,15 @@ struct PeerRpcReply : public messaging::Message { RequestVoteReply request_vote; AppendEntriesReply append_entries; - template - void serialize(Archive &ar) { - ar(cereal::virtual_base_class(this), type, request_vote, - append_entries); + private: + friend class boost::serialization::access; + + template + void serialize(TArchive &ar, unsigned int) { + ar &boost::serialization::base_object(*this); + ar &type; + ar &request_vote; + ar &append_entries; } }; diff --git a/src/communication/raft/raft.hpp b/src/communication/raft/raft.hpp index 94a46a139..d84f36a9b 100644 --- a/src/communication/raft/raft.hpp +++ b/src/communication/raft/raft.hpp @@ -10,9 +10,10 @@ #include #include +#include "boost/serialization/vector.hpp" #include "glog/logging.h" -#include "utils/cereal_optional.hpp" +#include "utils/serialization_optional.hpp" namespace communication::raft { @@ -43,9 +44,10 @@ struct LogEntry { } bool operator!=(const LogEntry &rhs) const { return !(*this == rhs); } - template - void serialize(Archive &ar) { - ar(term, command); + template + void serialize(TArchive &ar, unsigned int) { + ar &term; + ar &command; } }; @@ -56,9 +58,12 @@ struct RequestVoteRequest { LogIndex last_log_index; TermId last_log_term; - template - void serialize(Archive &ar) { - ar(candidate_term, candidate_id, last_log_index, last_log_term); + template + void serialize(TArchive &ar, unsigned int) { + ar &candidate_term; + ar &candidate_id; + ar &last_log_index; + ar &last_log_term; } }; @@ -66,9 +71,10 @@ struct RequestVoteReply { TermId term; bool vote_granted; - template - void serialize(Archive &ar) { - ar(term, vote_granted); + template + void serialize(TArchive &ar, unsigned int) { + ar &term; + ar &vote_granted; } }; @@ -81,10 +87,14 @@ struct AppendEntriesRequest { std::vector> entries; LogIndex leader_commit; - template - void serialize(Archive &ar) { - ar(leader_term, leader_id, prev_log_index, prev_log_term, entries, - leader_commit); + template + void serialize(TArchive &ar, unsigned int) { + ar &leader_term; + ar &leader_id; + ar &prev_log_index; + ar &prev_log_term; + ar &entries; + ar &leader_commit; } }; @@ -92,9 +102,10 @@ struct AppendEntriesReply { TermId term; bool success; - template - void serialize(Archive &ar) { - ar(term, success); + template + void serialize(TArchive &ar, unsigned int) { + ar &term; + ar &success; } }; diff --git a/src/communication/raft/test_utils.hpp b/src/communication/raft/test_utils.hpp index 7f27a0f0b..6258f72b9 100644 --- a/src/communication/raft/test_utils.hpp +++ b/src/communication/raft/test_utils.hpp @@ -10,12 +10,12 @@ struct DummyState { bool operator==(const Change &) const { return true; } bool operator!=(const Change &) const { return false; } - template - void serialize(Archive &ar) {} + template + void serialize(TArchive &, unsigned int) {} }; - template - void serialize(Archive &ar) {} + template + void serialize(TArchive &, unsigned int) {} }; struct IntState { @@ -31,15 +31,16 @@ struct IntState { } bool operator!=(const Change &rhs) const { return !(*this == rhs); }; - template - void serialize(Archive &ar) { - ar(t, d); + template + void serialize(TArchive &ar, unsigned int) { + ar &t; + ar &d; } }; - template - void serialize(Archive &ar) { - ar(x); + template + void serialize(TArchive &ar, unsigned int) { + ar &x; } }; @@ -52,21 +53,20 @@ class NoOpNetworkInterface : public RaftNetworkInterface { public: ~NoOpNetworkInterface() {} - virtual bool SendRequestVote(const MemberId &recipient, - const RequestVoteRequest &request, - RequestVoteReply &reply, - std::chrono::milliseconds timeout) override { + virtual bool SendRequestVote(const MemberId &, const RequestVoteRequest &, + RequestVoteReply &, + std::chrono::milliseconds) override { return false; } - virtual bool SendAppendEntries(const MemberId &recipient, - const AppendEntriesRequest &request, - AppendEntriesReply &reply, - std::chrono::milliseconds timeout) override { + virtual bool SendAppendEntries(const MemberId &, + const AppendEntriesRequest &, + AppendEntriesReply &, + std::chrono::milliseconds) override { return false; } - virtual void Start(RaftMember &member) override {} + virtual void Start(RaftMember &) override {} virtual void Shutdown() override {} }; @@ -80,10 +80,10 @@ class NextReplyNetworkInterface : public RaftNetworkInterface { public: ~NextReplyNetworkInterface() {} - virtual bool SendRequestVote(const MemberId &recipient, + virtual bool SendRequestVote(const MemberId &, const RequestVoteRequest &request, RequestVoteReply &reply, - std::chrono::milliseconds timeout) override { + std::chrono::milliseconds) override { PeerRpcRequest req; req.type = RpcType::REQUEST_VOTE; req.request_vote = request; @@ -97,10 +97,10 @@ class NextReplyNetworkInterface : public RaftNetworkInterface { return true; } - virtual bool SendAppendEntries(const MemberId &recipient, + virtual bool SendAppendEntries(const MemberId &, const AppendEntriesRequest &request, AppendEntriesReply &reply, - std::chrono::milliseconds timeout) override { + std::chrono::milliseconds) override { PeerRpcRequest req; req.type = RpcType::APPEND_ENTRIES; req.append_entries = request; @@ -114,7 +114,7 @@ class NextReplyNetworkInterface : public RaftNetworkInterface { return true; } - virtual void Start(RaftMember &member) override {} + virtual void Start(RaftMember &) override {} virtual void Shutdown() override {} diff --git a/src/communication/rpc/messages-inl.hpp b/src/communication/rpc/messages-inl.hpp new file mode 100644 index 000000000..551b0060f --- /dev/null +++ b/src/communication/rpc/messages-inl.hpp @@ -0,0 +1,35 @@ +#include "boost/archive/binary_iarchive.hpp" +#include "boost/archive/binary_oarchive.hpp" +#include "boost/serialization/export.hpp" + +#include "distributed/coordination_rpc_messages.hpp" +#include "storage/concurrent_id_mapper_rpc_messages.hpp" +#include "transactions/engine_rpc_messages.hpp" + +BOOST_CLASS_EXPORT(tx::SnapshotReq); +BOOST_CLASS_EXPORT(tx::SnapshotRes); +BOOST_CLASS_EXPORT(tx::GcSnapshotReq); +BOOST_CLASS_EXPORT(tx::ClogInfoReq); +BOOST_CLASS_EXPORT(tx::ClogInfoRes); +BOOST_CLASS_EXPORT(tx::ActiveTransactionsReq); +BOOST_CLASS_EXPORT(tx::IsActiveReq); +BOOST_CLASS_EXPORT(tx::IsActiveRes); + +#define ID_VALUE_EXPORT_BOOST_TYPE(type) \ + BOOST_CLASS_EXPORT(storage::type##IdReq); \ + BOOST_CLASS_EXPORT(storage::type##IdRes); \ + BOOST_CLASS_EXPORT(storage::Id##type##Req); \ + BOOST_CLASS_EXPORT(storage::Id##type##Res); + +ID_VALUE_EXPORT_BOOST_TYPE(Label) +ID_VALUE_EXPORT_BOOST_TYPE(EdgeType) +ID_VALUE_EXPORT_BOOST_TYPE(Property) + +#undef ID_VALUE_EXPORT_BOOST_TYPE + +BOOST_CLASS_EXPORT(distributed::RegisterWorkerReq); +BOOST_CLASS_EXPORT(distributed::RegisterWorkerRes); +BOOST_CLASS_EXPORT(distributed::GetEndpointReq); +BOOST_CLASS_EXPORT(distributed::GetEndpointRes); +BOOST_CLASS_EXPORT(distributed::StopWorkerReq); +BOOST_CLASS_EXPORT(distributed::StopWorkerRes); diff --git a/src/communication/rpc/rpc.cpp b/src/communication/rpc/rpc.cpp index bb1c25252..e4511c4a1 100644 --- a/src/communication/rpc/rpc.cpp +++ b/src/communication/rpc/rpc.cpp @@ -1,6 +1,13 @@ #include #include +#include "boost/archive/binary_iarchive.hpp" +#include "boost/archive/binary_oarchive.hpp" +#include "boost/serialization/access.hpp" +#include "boost/serialization/base_object.hpp" +#include "boost/serialization/export.hpp" +#include "boost/serialization/unique_ptr.hpp" + #include "communication/rpc/rpc.hpp" #include "utils/string.hpp" @@ -24,15 +31,19 @@ class Request : public messaging::Message { const std::string &message_id() const { return message_id_; } const messaging::Message &message() const { return *message_; } - template - void serialize(Archive &ar) { - ar(cereal::virtual_base_class(this), address_, port_, - stream_, message_id_, message_); - } + private: + friend class boost::serialization::access; + Request() {} // Needed for serialization. - protected: - friend class cereal::access; - Request() {} // Cereal needs access to a default constructor. + template + void serialize(TArchive &ar, unsigned int) { + ar &boost::serialization::base_object(*this); + ar &address_; + ar &port_; + ar &stream_; + ar &message_id_; + ar &message_; + } std::string address_; uint16_t port_; @@ -47,18 +58,20 @@ class Response : public messaging::Message { std::unique_ptr message) : message_id_(message_id), message_(std::move(message)) {} - template - void serialize(Archive &ar) { - ar(cereal::virtual_base_class(this), message_id_, - message_); - } - const auto &message_id() const { return message_id_; } auto &message() { return message_; } - protected: - Response() {} // Cereal needs access to a default constructor. - friend class cereal::access; + private: + friend class boost::serialization::access; + Response() {} // Needed for serialization. + + template + void serialize(TArchive &ar, unsigned int) { + ar &boost::serialization::base_object(*this); + ar &message_id_; + ar &message_; + } + std::string message_id_; std::unique_ptr message_; }; @@ -138,6 +151,8 @@ void Server::Shutdown() { stream_->Shutdown(); if (running_thread_.joinable()) running_thread_.join(); } + } // namespace communication::rpc -CEREAL_REGISTER_TYPE(communication::rpc::Request); -CEREAL_REGISTER_TYPE(communication::rpc::Response); + +BOOST_CLASS_EXPORT(communication::rpc::Request); +BOOST_CLASS_EXPORT(communication::rpc::Response); diff --git a/src/communication/rpc/rpc.hpp b/src/communication/rpc/rpc.hpp index 48e4cd9a6..021e52742 100644 --- a/src/communication/rpc/rpc.hpp +++ b/src/communication/rpc/rpc.hpp @@ -73,9 +73,8 @@ class Server { typename TRequestResponse::Response>::value, "TRequestResponse::Response must be derived from Message"); auto got = callbacks_.emplace( - typeid(typename TRequestResponse::Request), [callback = callback]( - const messaging::Message - &base_message) { + typeid(typename TRequestResponse::Request), + [callback = callback](const messaging::Message &base_message) { const auto &message = dynamic_cast( base_message); @@ -99,4 +98,5 @@ class Server { std::thread running_thread_; bool started_{false}; }; + } // namespace communication::rpc diff --git a/src/database/graph_db_datatypes.hpp b/src/database/graph_db_datatypes.hpp index b53deb3cf..8c136584d 100644 --- a/src/database/graph_db_datatypes.hpp +++ b/src/database/graph_db_datatypes.hpp @@ -3,7 +3,6 @@ #include #include "boost/serialization/base_object.hpp" -#include "cereal/types/base_class.hpp" #include "utils/total_ordering.hpp" @@ -28,21 +27,15 @@ class Common : TotalOrdering { size_t operator()(const TSpecificType &t) const { return hash(t.storage_); } }; - /** Required for cereal serialization. */ - template - void serialize(Archive &archive) { - archive(storage_); - } - private: - StorageT storage_{0}; - friend class boost::serialization::access; template void serialize(TArchive &ar, const unsigned int) { - ar & storage_; + ar &storage_; } + + StorageT storage_{0}; }; class Label : public Common