tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Cap'n Proto

Browse Source 
Summary:
There will be a lot of leftover files, execute the following commands inside
`src/` to remove them:
```
git clean -xf
rm -r rpc/ storage/single_node_ha/rpc/
```

Reviewers: teon.banek

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D2011
This commit is contained in:
Matej Ferencevic 2019-05-02 20:53:54 +02:00
parent 16d4a7b5b6
commit 5c244c1ad4
69 changed files with 379 additions and 3975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.gitignore vendored
View File

@ -34,71 +34,36 @@ TAGS
*.fas *.fas
*.fasl *.fasl
# Cap'n Proto generated files # LCP generated C++ files
*.capnp.c++
*.capnp.h
# LCP generated C++ & Cap'n Proto files
*.lcp.cpp *.lcp.cpp
# TODO delete later
src/storage/distributed/concurrent_id_mapper_rpc_messages.capnp
src/storage/distributed/concurrent_id_mapper_rpc_messages.hpp
#####
src/database/distributed/counters_rpc_messages.capnp
src/database/distributed/counters_rpc_messages.hpp src/database/distributed/counters_rpc_messages.hpp
src/database/distributed/serialization.capnp
src/database/distributed/serialization.hpp src/database/distributed/serialization.hpp
src/database/single_node_ha/serialization.capnp
src/database/single_node_ha/serialization.hpp src/database/single_node_ha/serialization.hpp
src/distributed/bfs_rpc_messages.capnp
src/distributed/bfs_rpc_messages.hpp src/distributed/bfs_rpc_messages.hpp
src/distributed/coordination_rpc_messages.capnp
src/distributed/coordination_rpc_messages.hpp src/distributed/coordination_rpc_messages.hpp
src/distributed/data_rpc_messages.capnp
src/distributed/data_rpc_messages.hpp src/distributed/data_rpc_messages.hpp
src/distributed/durability_rpc_messages.capnp
src/distributed/durability_rpc_messages.hpp src/distributed/durability_rpc_messages.hpp
src/distributed/dynamic_worker_rpc_messages.capnp
src/distributed/dynamic_worker_rpc_messages.hpp src/distributed/dynamic_worker_rpc_messages.hpp
src/distributed/index_rpc_messages.capnp
src/distributed/index_rpc_messages.hpp src/distributed/index_rpc_messages.hpp
src/distributed/plan_rpc_messages.capnp
src/distributed/plan_rpc_messages.hpp src/distributed/plan_rpc_messages.hpp
src/distributed/pull_produce_rpc_messages.capnp
src/distributed/pull_produce_rpc_messages.hpp src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.capnp
src/distributed/storage_gc_rpc_messages.hpp src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.capnp
src/distributed/token_sharing_rpc_messages.hpp src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.capnp
src/distributed/updates_rpc_messages.hpp src/distributed/updates_rpc_messages.hpp
src/query/frontend/ast/ast.hpp src/query/frontend/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.capnp
src/query/distributed/frontend/ast/ast_serialization.hpp src/query/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.capnp
src/durability/distributed/state_delta.hpp src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp src/query/frontend/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.capnp
src/query/distributed/frontend/semantic/symbol_serialization.hpp src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.capnp
src/query/distributed/plan/ops.hpp src/query/distributed/plan/ops.hpp
src/query/plan/operator.hpp src/query/plan/operator.hpp
src/raft/log_entry.capnp
src/raft/log_entry.hpp src/raft/log_entry.hpp
src/raft/raft_rpc_messages.capnp
src/raft/raft_rpc_messages.hpp src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.capnp
src/raft/snapshot_metadata.hpp src/raft/snapshot_metadata.hpp
src/raft/storage_info_rpc_messages.hpp src/raft/storage_info_rpc_messages.hpp
src/raft/storage_info_rpc_messages.capnp
src/stats/stats_rpc_messages.capnp
src/stats/stats_rpc_messages.hpp src/stats/stats_rpc_messages.hpp
src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp
src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp
src/transactions/distributed/engine_rpc_messages.capnp
src/transactions/distributed/engine_rpc_messages.hpp src/transactions/distributed/engine_rpc_messages.hpp

1
CMakeLists.txt
View File

@ -154,7 +154,6 @@ include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR})
include_directories(SYSTEM ${BZIP2_INCLUDE_DIR}) include_directories(SYSTEM ${BZIP2_INCLUDE_DIR})
include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR}) include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR})
include_directories(SYSTEM ${CAPNP_INCLUDE_DIR})
include_directories(SYSTEM ${LIBRDKAFKA_INCLUDE_DIR}) include_directories(SYSTEM ${LIBRDKAFKA_INCLUDE_DIR})
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------

24
cmake/functions.cmake
View File

@ -85,27 +85,3 @@ function(get_target_cxx_flags target result)
endif() endif()
set(${result} ${flags} PARENT_SCOPE) set(${result} ${flags} PARENT_SCOPE)
endfunction() endfunction()
# Define `add_capnp` function for registering a capnp file for generation.
#
# The `define_add_capnp` expects 3 arguments:
# * name -- name for the function, you usually want `add_capnp`
# * main_src_files -- variable to be updated with generated cpp files
# * generated_capnp_files -- variable to be updated with generated hpp and cpp files
#
# The `add_capnp` function expects a single argument, path to capnp file.
# Each added file is standalone and we avoid recompiling everything.
macro(define_add_capnp name main_src_files generated_capnp_files)
function(${name} capnp_src_file)
set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.c++)
set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h)
add_custom_command(OUTPUT ${cpp_file} ${h_file}
COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file} -I ${CMAKE_SOURCE_DIR}/src
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file} capnproto-proj
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
# Update *global* generated_capnp_files
set(${generated_capnp_files} ${${generated_capnp_files}} ${cpp_file} ${h_file} PARENT_SCOPE)
# Update *global* main_src_files
set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
endfunction(${name})
endmacro(define_add_capnp)

24
libs/CMakeLists.txt
View File

@ -213,30 +213,6 @@ import_external_library(rocksdb STATIC
DISABLE_WARNING_AS_ERROR=1 DISABLE_WARNING_AS_ERROR=1
INSTALL_COMMAND true) INSTALL_COMMAND true)
# Setup Cap'n Proto
ExternalProject_Add(capnproto-proj
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/configure
--prefix=${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local
--enable-shared=no --silent
CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
BUILD_COMMAND make -j${NPROC} check)
set(CAPNP_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/include
CACHE FILEPATH "Path to capnproto include directory" FORCE)
set(CAPNP_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libcapnp.a
CACHE FILEPATH "Path to capnproto library" FORCE)
set(KJ_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libkj.a
CACHE FILEPATH "Path to kj library (used by capnproto)" FORCE)
import_library(capnp STATIC ${CAPNP_LIBRARY} capnproto-proj)
import_library(kj STATIC ${KJ_LIBRARY} capnproto-proj)
set(CAPNP_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnp
CACHE FILEPATH "Path to capnproto executable" FORCE)
set(CAPNP_CXX_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnpc-c++
CACHE FILEPATH "Path to capnproto c++ plugin executable" FORCE)
mark_as_advanced(CAPNP_INCLUDE_DIR CAPNP_LIBRARY KJ_LIBRARY CAPNP_EXE CAPNP_CXX_EXE)
# Setup librdkafka. # Setup librdkafka.
import_external_library(librdkafka STATIC import_external_library(librdkafka STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a

7
libs/setup.sh
View File

@ -130,13 +130,6 @@ clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag
# fix compilation flags to work with clang 8 # fix compilation flags to work with clang 8
sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/Makefile sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/Makefile
# Cap'n Proto serialization (and RPC) lib
wget -nv http://deps.memgraph.io/capnproto-c++-0.6.1.tar.gz -O capnproto.tar.gz
tar -xzf capnproto.tar.gz
rm -rf capnproto
mv capnproto-c++-0.6.1 capnproto
rm capnproto.tar.gz
# kafka # kafka
kafka_tag="c319b4e987d0bc4fe4f01cf91419d90b62061655" # Mar 8, 2018 kafka_tag="c319b4e987d0bc4fe4f01cf91419d90b62061655" # Mar 8, 2018
# git clone https://github.com/edenhill/librdkafka.git # git clone https://github.com/edenhill/librdkafka.git

104
src/CMakeLists.txt
View File

@ -10,7 +10,6 @@ add_subdirectory(telemetry)
add_subdirectory(communication) add_subdirectory(communication)
add_subdirectory(stats) add_subdirectory(stats)
add_subdirectory(auth) add_subdirectory(auth)
add_subdirectory(rpc)
add_subdirectory(slk) add_subdirectory(slk)
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
@ -176,83 +175,51 @@ set(mg_distributed_sources
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
define_add_capnp(add_capnp_distributed mg_distributed_sources generated_capnp_files)
define_add_lcp(add_lcp_distributed mg_distributed_sources generated_lcp_distributed_files) define_add_lcp(add_lcp_distributed mg_distributed_sources generated_lcp_distributed_files)
add_lcp_distributed(durability/distributed/state_delta.lcp) add_lcp_distributed(durability/distributed/state_delta.lcp)
add_lcp_distributed(database/distributed/counters_rpc_messages.lcp CAPNP_SCHEMA @0x95a2c3ea3871e945) add_lcp_distributed(database/distributed/counters_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(database/distributed/counters_rpc_messages.capnp) add_lcp_distributed(database/distributed/serialization.lcp SLK_SERIALIZE
add_lcp_distributed(database/distributed/serialization.lcp CAPNP_SCHEMA @0xdea01657b3563887
DEPENDS durability/distributed/state_delta.lcp) DEPENDS durability/distributed/state_delta.lcp)
add_capnp_distributed(database/distributed/serialization.capnp) add_lcp_distributed(distributed/bfs_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_distributed(distributed/bfs_rpc_messages.lcp CAPNP_SCHEMA @0x8e508640b09b6d2a) add_lcp_distributed(distributed/coordination_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(distributed/bfs_rpc_messages.capnp) add_lcp_distributed(distributed/data_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_distributed(distributed/coordination_rpc_messages.lcp CAPNP_SCHEMA @0x93df0c4703cf98fb) add_lcp_distributed(distributed/durability_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(distributed/coordination_rpc_messages.capnp) add_lcp_distributed(distributed/index_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_distributed(distributed/data_rpc_messages.lcp CAPNP_SCHEMA @0xc1c8a341ba37aaf5) add_lcp_distributed(distributed/plan_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(distributed/data_rpc_messages.capnp) add_lcp_distributed(distributed/pull_produce_rpc_messages.lcp SLK_SERIALIZE
add_lcp_distributed(distributed/durability_rpc_messages.lcp CAPNP_SCHEMA @0xf5e53bc271e2163d)
add_capnp_distributed(distributed/durability_rpc_messages.capnp)
add_lcp_distributed(distributed/index_rpc_messages.lcp CAPNP_SCHEMA @0xa8aab46862945bd6)
add_capnp_distributed(distributed/index_rpc_messages.capnp)
add_lcp_distributed(distributed/plan_rpc_messages.lcp CAPNP_SCHEMA @0xfcbc48dc9f106d28)
add_capnp_distributed(distributed/plan_rpc_messages.capnp)
add_lcp_distributed(distributed/pull_produce_rpc_messages.lcp CAPNP_SCHEMA @0xa78a9254a73685bd
DEPENDS transactions/distributed/serialization.lcp) DEPENDS transactions/distributed/serialization.lcp)
add_capnp_distributed(distributed/pull_produce_rpc_messages.capnp) add_lcp_distributed(distributed/storage_gc_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_distributed(distributed/storage_gc_rpc_messages.lcp CAPNP_SCHEMA @0xd705663dfe36cf81) add_lcp_distributed(distributed/token_sharing_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(distributed/storage_gc_rpc_messages.capnp) add_lcp_distributed(distributed/updates_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_distributed(distributed/token_sharing_rpc_messages.lcp CAPNP_SCHEMA @0x8f295db54ec4caec) add_lcp_distributed(distributed/dynamic_worker_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(distributed/token_sharing_rpc_messages.capnp)
add_lcp_distributed(distributed/updates_rpc_messages.lcp CAPNP_SCHEMA @0x82d5f38d73c7b53a)
add_capnp_distributed(distributed/updates_rpc_messages.capnp)
add_lcp_distributed(distributed/dynamic_worker_rpc_messages.lcp CAPNP_SCHEMA @0x8c53f6c9a0c71b05)
add_capnp_distributed(distributed/dynamic_worker_rpc_messages.capnp)
# distributed_ops.lcp is leading the capnp code generation, so we don't need
# to generate any capnp for operator.lcp
add_lcp_distributed(query/frontend/ast/ast.lcp) add_lcp_distributed(query/frontend/ast/ast.lcp)
add_lcp_distributed(query/distributed/frontend/ast/ast_serialization.lcp CAPNP_SCHEMA @0xb107d3d6b4b1600b add_lcp_distributed(query/distributed/frontend/ast/ast_serialization.lcp SLK_SERIALIZE
DEPENDS query/frontend/ast/ast.lcp) DEPENDS query/frontend/ast/ast.lcp)
add_capnp_distributed(query/distributed/frontend/ast/ast_serialization.capnp)
add_lcp_distributed(query/frontend/semantic/symbol.lcp) add_lcp_distributed(query/frontend/semantic/symbol.lcp)
add_lcp_distributed(query/distributed/frontend/semantic/symbol_serialization.lcp CAPNP_SCHEMA @0x93c1dcee84e93b76 add_lcp_distributed(query/distributed/frontend/semantic/symbol_serialization.lcp SLK_SERIALIZE
DEPENDS query/frontend/semantic/symbol.lcp) DEPENDS query/frontend/semantic/symbol.lcp)
add_lcp_distributed(query/plan/operator.lcp) add_lcp_distributed(query/plan/operator.lcp)
add_lcp_distributed(query/distributed/plan/ops.lcp CAPNP_SCHEMA @0xe5cae8d045d30c42 add_lcp_distributed(query/distributed/plan/ops.lcp SLK_SERIALIZE
DEPENDS query/plan/operator.lcp) DEPENDS query/plan/operator.lcp)
add_capnp_distributed(query/distributed/plan/ops.capnp)
add_lcp_distributed(storage/distributed/rpc/concurrent_id_mapper_rpc_messages.lcp CAPNP_SCHEMA @0xa6068dae93d225dd) add_lcp_distributed(storage/distributed/rpc/concurrent_id_mapper_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_distributed(storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp) add_lcp_distributed(transactions/distributed/engine_rpc_messages.lcp SLK_SERIALIZE
add_lcp_distributed(transactions/distributed/engine_rpc_messages.lcp CAPNP_SCHEMA @0xde02b7c49180cad5
DEPENDS transactions/distributed/serialization.lcp) DEPENDS transactions/distributed/serialization.lcp)
add_capnp_distributed(transactions/distributed/engine_rpc_messages.capnp)
add_custom_target(generate_lcp_distributed DEPENDS ${generated_lcp_distributed_files}) add_custom_target(generate_lcp_distributed DEPENDS ${generated_lcp_distributed_files})
# Registering capnp must come after registering lcp files.
add_capnp_distributed(communication/rpc/messages.capnp)
add_capnp_distributed(durability/distributed/serialization.capnp)
add_capnp_distributed(query/distributed/frontend/semantic/symbol_serialization.capnp)
add_capnp_distributed(query/distributed/serialization.capnp)
add_capnp_distributed(storage/distributed/rpc/serialization.capnp)
add_custom_target(generate_capnp DEPENDS generate_lcp_distributed ${generated_capnp_files})
set(MG_DISTRIBUTED_LIBS stdc++fs Threads::Threads fmt cppitertools set(MG_DISTRIBUTED_LIBS stdc++fs Threads::Threads fmt cppitertools
antlr_opencypher_parser_lib dl glog gflags capnp kj antlr_opencypher_parser_lib dl glog gflags
mg-utils mg-io mg-io-serialization mg-integrations-kafka mg-requests mg-utils mg-io mg-integrations-kafka mg-requests
mg-communication mg-comm-rpc mg-auth mg-rpc) mg-communication mg-comm-rpc mg-auth)
# STATIC library used by memgraph executables # STATIC library used by memgraph executables
add_library(mg-distributed STATIC ${mg_distributed_sources}) add_library(mg-distributed STATIC ${mg_distributed_sources})
target_link_libraries(mg-distributed ${MG_DISTRIBUTED_LIBS}) target_link_libraries(mg-distributed ${MG_DISTRIBUTED_LIBS})
add_dependencies(mg-distributed generate_opencypher_parser) add_dependencies(mg-distributed generate_opencypher_parser)
add_dependencies(mg-distributed generate_lcp_distributed) add_dependencies(mg-distributed generate_lcp_distributed)
add_dependencies(mg-distributed generate_capnp)
target_compile_definitions(mg-distributed PUBLIC MG_DISTRIBUTED) target_compile_definitions(mg-distributed PUBLIC MG_DISTRIBUTED)
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
@ -300,44 +267,30 @@ set(mg_single_node_ha_sources
storage/common/locking/record_lock.cpp storage/common/locking/record_lock.cpp
storage/single_node_ha/edge_accessor.cpp storage/single_node_ha/edge_accessor.cpp
storage/single_node_ha/record_accessor.cpp storage/single_node_ha/record_accessor.cpp
storage/single_node_ha/rpc/serialization.cpp
storage/single_node_ha/vertex_accessor.cpp storage/single_node_ha/vertex_accessor.cpp
transactions/single_node_ha/engine.cpp transactions/single_node_ha/engine.cpp
memgraph_init.cpp memgraph_init.cpp
) )
define_add_capnp(add_capnp_single_node_ha mg_single_node_ha_sources generated_capnp_single_node_ha_files)
define_add_lcp(add_lcp_single_node_ha mg_single_node_ha_sources generated_lcp_single_node_ha_files) define_add_lcp(add_lcp_single_node_ha mg_single_node_ha_sources generated_lcp_single_node_ha_files)
add_lcp_single_node_ha(durability/single_node_ha/state_delta.lcp) add_lcp_single_node_ha(durability/single_node_ha/state_delta.lcp)
add_lcp_single_node_ha(database/single_node_ha/serialization.lcp CAPNP_SCHEMA @0xd0f4a502575fb6f7 add_lcp_single_node_ha(database/single_node_ha/serialization.lcp SLK_SERIALIZE
DEPENDS durability/single_node_ha/state_delta.lcp) DEPENDS durability/single_node_ha/state_delta.lcp)
add_capnp_single_node_ha(database/single_node_ha/serialization.capnp)
add_lcp_single_node_ha(query/frontend/ast/ast.lcp) add_lcp_single_node_ha(query/frontend/ast/ast.lcp)
add_lcp_single_node_ha(query/frontend/semantic/symbol.lcp) add_lcp_single_node_ha(query/frontend/semantic/symbol.lcp)
add_lcp_single_node_ha(query/plan/operator.lcp) add_lcp_single_node_ha(query/plan/operator.lcp)
add_lcp_single_node_ha(raft/raft_rpc_messages.lcp CAPNP_SCHEMA @0xa6c29b4287233b66) add_lcp_single_node_ha(raft/raft_rpc_messages.lcp SLK_SERIALIZE)
add_capnp_single_node_ha(raft/raft_rpc_messages.capnp) add_lcp_single_node_ha(raft/log_entry.lcp SLK_SERIALIZE)
add_lcp_single_node_ha(raft/log_entry.lcp CAPNP_SCHEMA @0x96c07fe13850c22a) add_lcp_single_node_ha(raft/snapshot_metadata.lcp SLK_SERIALIZE)
add_capnp_single_node_ha(raft/log_entry.capnp) add_lcp_single_node_ha(raft/storage_info_rpc_messages.lcp SLK_SERIALIZE)
add_lcp_single_node_ha(raft/snapshot_metadata.lcp CAPNP_SCHEMA @0xaa08e34991680f6c)
add_capnp_single_node_ha(raft/snapshot_metadata.capnp)
add_lcp_single_node_ha(raft/storage_info_rpc_messages.lcp CAPNP_SCHEMA @0xceee3960cd8eaa7e)
add_capnp_single_node_ha(raft/storage_info_rpc_messages.capnp)
add_custom_target(generate_lcp_single_node_ha DEPENDS ${generated_lcp_single_node_ha_files}) add_custom_target(generate_lcp_single_node_ha DEPENDS ${generated_lcp_single_node_ha_files})
# Registering capnp must come after registering lcp files.
add_capnp_single_node_ha(storage/single_node_ha/rpc/serialization.capnp)
add_custom_target(generate_capnp_single_node_ha DEPENDS generate_lcp_single_node_ha ${generated_capnp_single_node_ha_files})
set(MG_SINGLE_NODE_HA_LIBS stdc++fs Threads::Threads fmt cppitertools set(MG_SINGLE_NODE_HA_LIBS stdc++fs Threads::Threads fmt cppitertools
antlr_opencypher_parser_lib dl glog gflags capnp kj antlr_opencypher_parser_lib dl glog gflags
mg-utils mg-io mg-integrations-kafka mg-requests mg-communication mg-comm-rpc mg-utils mg-io mg-integrations-kafka mg-requests mg-communication mg-comm-rpc
mg-auth mg-rpc) mg-auth)
if (USE_LTALLOC) if (USE_LTALLOC)
list(APPEND MG_SINGLE_NODE_HA_LIBS ltalloc) list(APPEND MG_SINGLE_NODE_HA_LIBS ltalloc)
@ -353,7 +306,6 @@ add_library(mg-single-node-ha STATIC ${mg_single_node_ha_sources})
target_link_libraries(mg-single-node-ha ${MG_SINGLE_NODE_HA_LIBS}) target_link_libraries(mg-single-node-ha ${MG_SINGLE_NODE_HA_LIBS})
add_dependencies(mg-single-node-ha generate_opencypher_parser) add_dependencies(mg-single-node-ha generate_opencypher_parser)
add_dependencies(mg-single-node-ha generate_lcp_single_node_ha) add_dependencies(mg-single-node-ha generate_lcp_single_node_ha)
add_dependencies(mg-single-node-ha generate_capnp_single_node_ha)
target_compile_definitions(mg-single-node-ha PUBLIC MG_SINGLE_NODE_HA) target_compile_definitions(mg-single-node-ha PUBLIC MG_SINGLE_NODE_HA)
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------

11
src/communication/CMakeLists.txt
View File

@ -16,15 +16,6 @@ set(communication_rpc_src_files
rpc/protocol.cpp rpc/protocol.cpp
rpc/server.cpp) rpc/server.cpp)
define_add_capnp(add_capnp communication_rpc_src_files communication_rpc_capnp_files)
add_capnp(rpc/messages.capnp)
add_custom_target(generate_communication_rpc_capnp DEPENDS ${communication_rpc_capnp_files})
add_library(mg-comm-rpc STATIC ${communication_rpc_src_files}) add_library(mg-comm-rpc STATIC ${communication_rpc_src_files})
target_link_libraries(mg-comm-rpc Threads::Threads mg-communication mg-utils mg-io mg-rpc fmt glog gflags) target_link_libraries(mg-comm-rpc Threads::Threads mg-communication mg-utils mg-io fmt glog gflags)
target_link_libraries(mg-comm-rpc capnp kj)
target_link_libraries(mg-comm-rpc mg-slk) target_link_libraries(mg-comm-rpc mg-slk)
add_dependencies(mg-comm-rpc generate_communication_rpc_capnp)

2
src/communication/rpc/client.hpp
View File

@ -4,8 +4,6 @@
#include <mutex> #include <mutex>
#include <optional> #include <optional>
#include <capnp/message.h>
#include <capnp/serialize.h>
#include <glog/logging.h> #include <glog/logging.h>
#include "communication/client.hpp" #include "communication/client.hpp"

9
src/communication/rpc/messages.capnp
View File

@ -1,9 +0,0 @@
@0xd3832c9a1a3d8ec7;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("communication::rpc::capnp");
struct Message {
typeId @0 :UInt64;
data @1 :AnyPointer;
}

3
src/database/distributed/counters_rpc_messages.lcp
View File

@ -4,14 +4,11 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "database/distributed/counters_rpc_messages.capnp.h"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
cpp<# cpp<#
(lcp:namespace database) (lcp:namespace database)
(lcp:capnp-namespace "database")
(lcp:define-rpc counters-get (lcp:define-rpc counters-get
(:request ((name "std::string"))) (:request ((name "std::string")))
(:response ((value :int64_t)))) (:response ((value :int64_t))))

1
src/database/distributed/serialization.lcp
View File

@ -1,7 +1,6 @@
#>cpp #>cpp
#pragma once #pragma once
#include "database/distributed/serialization.capnp.h"
#include "durability/distributed/state_delta.hpp" #include "durability/distributed/state_delta.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
cpp<# cpp<#

3
src/database/single_node_ha/serialization.lcp
View File

@ -1,9 +1,8 @@
#>cpp #>cpp
#pragma once #pragma once
#include "database/single_node_ha/serialization.capnp.h"
#include "durability/single_node_ha/state_delta.hpp" #include "durability/single_node_ha/state_delta.hpp"
#include "storage/single_node_ha/rpc/serialization.hpp" #include "storage/common/types/slk.hpp"
cpp<# cpp<#
;; Generate serialization of state-delta ;; Generate serialization of state-delta

157
src/distributed/bfs_rpc_messages.lcp
View File

@ -4,92 +4,34 @@
#include <tuple> #include <tuple>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/bfs_rpc_messages.capnp.h"
#include "distributed/bfs_subcursor.hpp" #include "distributed/bfs_subcursor.hpp"
#include "query/frontend/semantic/symbol_table.hpp" #include "query/frontend/semantic/symbol_table.hpp"
#include "query/distributed/plan/ops.hpp" #include "query/distributed/plan/ops.hpp"
#include "query/distributed/serialization.hpp" #include "query/distributed/serialization.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
#include "transactions/type.hpp" #include "transactions/type.hpp"
#include "rpc/serialization.hpp"
cpp<# cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'ast "/query/distributed/frontend/ast/ast_serialization.capnp")
(lcp:capnp-import 'dist-ops "/query/distributed/plan/ops.capnp")
(lcp:capnp-import 'query "/query/distributed/serialization.capnp")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:capnp-type-conversion "storage::EdgeAddress" "Storage.Address")
(lcp:capnp-type-conversion "storage::VertexAddress" "Storage.Address")
(lcp:capnp-type-conversion "storage::EdgeType" "Storage.EdgeType")
(lcp:define-rpc create-bfs-subcursor (lcp:define-rpc create-bfs-subcursor
(:request (:request
((tx-id "tx::TransactionId" :capnp-type "UInt64") ((tx-id "tx::TransactionId")
(direction "query::EdgeAtom::Direction" (direction "query::EdgeAtom::Direction")
:capnp-type "Ast.EdgeAtom.Direction" :capnp-init nil (edge-types "std::vector<storage::EdgeType>")
:capnp-save (lcp:capnp-save-enum "::query::capnp::EdgeAtom::Direction"
"query::EdgeAtom::Direction"
'(in out both))
:capnp-load (lcp:capnp-load-enum "::query::capnp::EdgeAtom::Direction"
"query::EdgeAtom::Direction"
'(in out both)))
(edge-types "std::vector<storage::EdgeType>"
:capnp-save (lcp:capnp-save-vector "::storage::capnp::EdgeType"
"storage::EdgeType")
:capnp-load (lcp:capnp-load-vector "::storage::capnp::EdgeType"
"storage::EdgeType"))
(filter-lambda "query::plan::ExpansionLambda" (filter-lambda "query::plan::ExpansionLambda"
:slk-load (lambda (member) :slk-load (lambda (member)
#>cpp #>cpp
slk::Load(&self->${member}, reader, ast_storage); slk::Load(&self->${member}, reader, ast_storage);
cpp<#) cpp<#))
:capnp-type "DistOps.ExpansionLambda" (symbol-table "query::SymbolTable")
:capnp-load (lambda (reader member capnp-name)
#>cpp
Load(&${member}, ${reader}, ast_storage);
cpp<#))
(symbol-table "query::SymbolTable" :capnp-type "Query.SymbolTable")
(timestamp :int64_t) (timestamp :int64_t)
(parameters "query::Parameters" (parameters "query::Parameters"))
:capnp-type "Utils.Map(Utils.BoxInt64, Storage.PropertyValue)")) (:serialize (:slk :load-args '((ast-storage "query::AstStorage *")))))
(:serialize (:slk :load-args '((ast-storage "query::AstStorage *")))
(:capnp :load-args '((ast-storage "query::AstStorage *")))))
(:response ((member :int64_t)))) (:response ((member :int64_t))))
(lcp:define-rpc register-subcursors (lcp:define-rpc register-subcursors
(:request ((subcursor-ids "std::unordered_map<int16_t, int64_t>" (:request ((subcursor-ids "std::unordered_map<int16_t, int64_t>")))
:capnp-type "Utils.Map(Utils.BoxInt16, Utils.BoxInt64)"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveMap<utils::capnp::BoxInt16, utils::capnp::BoxInt64>(
${member}, &${builder},
[](auto *builder, const auto &entry) {
auto key_builder = builder->initKey();
key_builder.setValue(entry.first);
auto value_builder = builder->initValue();
value_builder.setValue(entry.second);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::LoadMap<utils::capnp::BoxInt16, utils::capnp::BoxInt64>(
&${member}, ${reader},
[](const auto &reader) {
int16_t key = reader.getKey().getValue();
int64_t value = reader.getValue().getValue();
return std::make_pair(key, value);
});
cpp<#))))
(:response ())) (:response ()))
(lcp:define-rpc reset-subcursor (lcp:define-rpc reset-subcursor
@ -123,27 +65,10 @@ cpp<#
if (has_value) { if (has_value) {
self->${member} = slk::LoadVertexAccessor(reader, dba, data_manager); self->${member} = slk::LoadVertexAccessor(reader, dba, data_manager);
} }
cpp<#) cpp<#)))
:capnp-type "Utils.Optional(Storage.VertexAccessor)"
:capnp-save (lcp:capnp-save-optional
"storage::capnp::VertexAccessor"
"VertexAccessor"
"[worker_id](auto *builder, const auto &vertex) {
storage::SaveVertexAccessor(vertex, builder, storage::SendVersions::BOTH, worker_id);
}")
:capnp-load (lcp:capnp-load-optional
"storage::capnp::VertexAccessor"
"VertexAccessor"
"[dba, data_manager](const auto &reader) {
return storage::LoadVertexAccessor(reader, dba, data_manager);
}")))
(:serialize (:slk :save-args '((worker-id :int16_t)) (:serialize (:slk :save-args '((worker-id :int16_t))
:load-args '((dba "database::GraphDbAccessor *") :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))) (data-manager "distributed::DataManager *"))))))
(:capnp
:save-args '((worker-id :int16_t))
:load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))))))
(lcp:define-rpc set-source (lcp:define-rpc set-source
(:request (:request
@ -161,16 +86,10 @@ cpp<#
(lcp:define-rpc reconstruct-path (lcp:define-rpc reconstruct-path
(:request (:request
((subcursor-id :int64_t) ((subcursor-id :int64_t)
(vertex "std::optional<storage::VertexAddress>" (vertex "std::optional<storage::VertexAddress>")
:capnp-save (lcp:capnp-save-optional "storage::capnp::Address" "storage::VertexAddress") (edge "std::optional<storage::EdgeAddress>"))
:capnp-load (lcp:capnp-load-optional "storage::capnp::Address" "storage::VertexAddress"))
(edge "std::optional<storage::EdgeAddress>"
:capnp-save (lcp:capnp-save-optional "storage::capnp::Address" "storage::EdgeAddress")
:capnp-load (lcp:capnp-load-optional "storage::capnp::Address" "storage::EdgeAddress")))
(:public (:public
#>cpp #>cpp
using Capnp = capnp::ReconstructPathReq;
ReconstructPathReq() {} ReconstructPathReq() {}
ReconstructPathReq(int64_t subcursor_id, storage::VertexAddress vertex) ReconstructPathReq(int64_t subcursor_id, storage::VertexAddress vertex)
@ -201,35 +120,14 @@ cpp<#
for (size_t i = 0; i < size; ++i) { for (size_t i = 0; i < size; ++i) {
self->${member}.push_back(slk::LoadEdgeAccessor(reader, dba, data_manager)); self->${member}.push_back(slk::LoadEdgeAccessor(reader, dba, data_manager));
} }
cpp<#) cpp<#))
:capnp-type "List(Storage.EdgeAccessor)" (next-vertex "std::optional<storage::VertexAddress>")
:capnp-save (lcp:capnp-save-vector "storage::capnp::EdgeAccessor" (next-edge "std::optional<storage::EdgeAddress>"))
"EdgeAccessor"
"[worker_id](auto *builder, const auto &edge) {
storage::SaveEdgeAccessor(edge, builder, storage::SendVersions::BOTH, worker_id);
}")
:capnp-load (lcp:capnp-load-vector "storage::capnp::EdgeAccessor"
"EdgeAccessor"
"[dba, data_manager](const auto &reader) {
return storage::LoadEdgeAccessor(reader, dba, data_manager);
}"))
(next-vertex "std::optional<storage::VertexAddress>"
:capnp-save (lcp:capnp-save-optional "storage::capnp::Address" "storage::VertexAddress")
:capnp-load (lcp:capnp-load-optional "storage::capnp::Address" "storage::VertexAddress"))
(next-edge "std::optional<storage::EdgeAddress>"
:capnp-save (lcp:capnp-save-optional "storage::capnp::Address" "storage::EdgeAddress")
:capnp-load (lcp:capnp-load-optional "storage::capnp::Address" "storage::EdgeAddress")))
(:serialize (:slk :save-args '((worker-id :int16_t)) (:serialize (:slk :save-args '((worker-id :int16_t))
:load-args '((dba "database::GraphDbAccessor *") :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))) (data-manager "distributed::DataManager *"))))
(:capnp
:save-args '((worker-id :int16_t))
:load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))))
(:public (:public
#>cpp #>cpp
using Capnp = capnp::ReconstructPathRes;
ReconstructPathRes() {} ReconstructPathRes() {}
ReconstructPathRes( ReconstructPathRes(
@ -264,29 +162,10 @@ cpp<#
for (size_t i = 0; i < size; ++i) { for (size_t i = 0; i < size; ++i) {
slk::Load(&self->${member}[i], reader, subcursor->db_accessor(), data_manager); slk::Load(&self->${member}[i], reader, subcursor->db_accessor(), data_manager);
} }
cpp<#) cpp<#))
:capnp-type "List(Query.TypedValue)"
:capnp-save (lcp:capnp-save-vector
"query::capnp::TypedValue"
"query::TypedValue"
"[&self](auto *builder, const auto &value) {
query::SaveCapnpTypedValue(value, builder,
storage::SendVersions::ONLY_OLD, self.worker_id);
}")
:capnp-load (lcp:capnp-load-vector
"query::capnp::TypedValue"
"query::TypedValue"
"[dba, data_manager](const auto &reader) {
query::TypedValue value;
query::LoadCapnpTypedValue(reader, &value, dba, data_manager);
return value;
}"))
(worker-id :int16_t :dont-save t)) (worker-id :int16_t :dont-save t))
(:serialize (:slk :load-args '((subcursor_storage "distributed::BfsSubcursorStorage *") (:serialize (:slk :load-args '((subcursor_storage "distributed::BfsSubcursorStorage *")
(data-manager "distributed::DataManager *"))) (data-manager "distributed::DataManager *")))))
(:capnp
:load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *")))))
(:response ())) (:response ()))
(lcp:pop-namespace) ;; distributed (lcp:pop-namespace) ;; distributed

64
src/distributed/coordination_rpc_messages.lcp
View File

@ -5,7 +5,6 @@
#include <unordered_map> #include <unordered_map>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/coordination_rpc_messages.capnp.h"
#include "durability/distributed/recovery.hpp" #include "durability/distributed/recovery.hpp"
#include "durability/distributed/serialization.hpp" #include "durability/distributed/serialization.hpp"
#include "io/network/endpoint.hpp" #include "io/network/endpoint.hpp"
@ -14,12 +13,6 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'dur "/durability/distributed/serialization.capnp")
(lcp:capnp-import 'io "/io/network/endpoint.capnp")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:define-rpc register-worker (lcp:define-rpc register-worker
(:request (:request
((desired-worker-id :int16_t) ((desired-worker-id :int16_t)
@ -28,61 +21,13 @@ cpp<#
(:response (:response
((registration-successful :bool) ((registration-successful :bool)
(durability-error :bool) (durability-error :bool)
(snapshot-to-recover "std::optional<std::pair<int64_t, tx::TransactionId>>" (snapshot-to-recover "std::optional<std::pair<int64_t, tx::TransactionId>>")
:capnp-type "Utils.Optional(Utils.Pair(Utils.BoxUInt64, Utils.BoxUInt64))" (workers "std::unordered_map<int, io::network::Endpoint>"))))
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveOptional<utils::capnp::Pair<utils::capnp::BoxUInt64, utils::capnp::BoxUInt64>, std::pair<int64_t, tx::TransactionId>>(
${member}, &${builder},
[](auto builder, const auto &v) {
auto first_builder = builder->initFirst();
auto second_builder = builder->initSecond();
first_builder.setValue(v.first);
second_builder.setValue(v.second);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
${member} = utils::LoadOptional<utils::capnp::Pair<utils::capnp::BoxUInt64, utils::capnp::BoxUInt64>, std::pair<int64_t, tx::TransactionId>>(
${reader}, [](auto reader){
return std::make_pair(reader.getFirst().getValue(), reader.getSecond().getValue());
});
cpp<#))
(workers "std::unordered_map<int, io::network::Endpoint>"
:capnp-type "Utils.Map(Utils.BoxInt16, Io.Endpoint)"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveMap<utils::capnp::BoxInt16, io::network::capnp::Endpoint>(
${member}, &${builder}, [](auto *builder, const auto &entry) {
auto key_builder = builder->initKey();
key_builder.setValue(entry.first);
auto value_builder = builder->initValue();
Save(entry.second, &value_builder);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::LoadMap<utils::capnp::BoxInt16, io::network::capnp::Endpoint>(&${member}, ${reader},
[](const auto &reader) {
io::network::Endpoint value;
Load(&value, reader.getValue());
return std::make_pair(
reader.getKey().getValue(), value);
});
cpp<#)))))
(lcp:define-rpc cluster-discovery (lcp:define-rpc cluster-discovery
(:request (:request
((worker-id :int16_t) ((worker-id :int16_t)
(endpoint "io::network::Endpoint" :capnp-type "Io.Endpoint"))) (endpoint "io::network::Endpoint")))
(:response ())) (:response ()))
(lcp:define-rpc stop-worker (lcp:define-rpc stop-worker
@ -92,8 +37,7 @@ cpp<#
(lcp:define-rpc notify-worker-recovered (lcp:define-rpc notify-worker-recovered
(:request (:request
((worker-id :int16_t) ((worker-id :int16_t)
(recovery-info "std::optional<durability::RecoveryInfo>" (recovery-info "std::optional<durability::RecoveryInfo>")))
:capnp-type "Utils.Optional(Dur.RecoveryInfo)")))
(:response ())) (:response ()))
(lcp:define-rpc heartbeat (lcp:define-rpc heartbeat

94
src/distributed/data_rpc_messages.lcp
View File

@ -5,7 +5,6 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/data_rpc_messages.capnp.h"
#include "storage/distributed/edge.hpp" #include "storage/distributed/edge.hpp"
#include "storage/distributed/gid.hpp" #include "storage/distributed/gid.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
@ -15,33 +14,17 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:define-struct tx-gid-pair () (lcp:define-struct tx-gid-pair ()
((tx-id "tx::TransactionId" :capnp-type "UInt64") ((tx-id "tx::TransactionId")
(gid "gid::Gid" :capnp-type "UInt64") (gid "gid::Gid")
(from-worker-id :int64_t)) (from-worker-id :int64_t))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-rpc vertex (lcp:define-rpc vertex
(:request ((member "TxGidPair"))) (:request ((member "TxGidPair")))
(:response (:response
((cypher-id :int64_t) ((cypher-id :int64_t)
(vertex-old-input "const Vertex *" (vertex-old-input "const Vertex *"
:capnp-type "Storage.Vertex"
:capnp-init nil
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
if (${member}) {
auto my_builder = ${builder}->initVertexOldInput();
storage::SaveVertex(*${member}, &my_builder, self.worker_id);
}
cpp<#)
:slk-save :slk-save
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -51,15 +34,6 @@ cpp<#
slk::Save(*self.${member}, builder, self.worker_id); slk::Save(*self.${member}, builder, self.worker_id);
} }
cpp<#) cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore member capnp-name))
#>cpp
if (${reader}.hasVertexOldInput()) {
auto my_reader = ${reader}.getVertexOldInput();
self->vertex_old_output = storage::LoadVertex(my_reader);
}
cpp<#)
:slk-load :slk-load
(lambda (member) (lambda (member)
(declare (ignore member)) (declare (ignore member))
@ -72,17 +46,6 @@ cpp<#
} }
cpp<#)) cpp<#))
(vertex-new-input "const Vertex *" (vertex-new-input "const Vertex *"
:capnp-type "Storage.Vertex"
:capnp-init nil
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
if (${member}) {
auto my_builder = ${builder}->initVertexNewInput();
storage::SaveVertex(*${member}, &my_builder, self.worker_id);
}
cpp<#)
:slk-save :slk-save
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -92,15 +55,6 @@ cpp<#
slk::Save(*self.${member}, builder, self.worker_id); slk::Save(*self.${member}, builder, self.worker_id);
} }
cpp<#) cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore member capnp-name))
#>cpp
if (${reader}.hasVertexNewInput()) {
auto my_reader = ${reader}.getVertexNewInput();
self->vertex_new_output = storage::LoadVertex(my_reader);
}
cpp<#)
:slk-load :slk-load
(lambda (member) (lambda (member)
(declare (ignore member)) (declare (ignore member))
@ -121,17 +75,6 @@ cpp<#
(:response (:response
((cypher-id :int64_t) ((cypher-id :int64_t)
(edge-old-input "const Edge *" (edge-old-input "const Edge *"
:capnp-type "Storage.Edge"
:capnp-init nil
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
if (${member}) {
auto my_builder = ${builder}->initEdgeOldInput();
storage::SaveEdge(*${member}, &my_builder, self.worker_id);
}
cpp<#)
:slk-save :slk-save
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -141,15 +84,6 @@ cpp<#
slk::Save(*self.${member}, builder, self.worker_id); slk::Save(*self.${member}, builder, self.worker_id);
} }
cpp<#) cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore member capnp-name))
#>cpp
if (${reader}.hasEdgeOldInput()) {
auto my_reader = ${reader}.getEdgeOldInput();
self->edge_old_output = storage::LoadEdge(my_reader);
}
cpp<#)
:slk-load :slk-load
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -160,17 +94,6 @@ cpp<#
} }
cpp<#)) cpp<#))
(edge-new-input "const Edge *" (edge-new-input "const Edge *"
:capnp-type "Storage.Edge"
:capnp-init nil
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
if (${member}) {
auto my_builder = ${builder}->initEdgeNewInput();
storage::SaveEdge(*${member}, &my_builder, self.worker_id);
}
cpp<#)
:slk-save :slk-save
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -180,15 +103,6 @@ cpp<#
slk::Save(*self.${member}, builder, self.worker_id); slk::Save(*self.${member}, builder, self.worker_id);
} }
cpp<#) cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore member capnp-name))
#>cpp
if (${reader}.hasEdgeNewInput()) {
auto my_reader = ${reader}.getEdgeNewInput();
self->edge_new_output = storage::LoadEdge(my_reader);
}
cpp<#)
:slk-load :slk-load
(lambda (member) (lambda (member)
#>cpp #>cpp
@ -203,7 +117,7 @@ cpp<#
(edge-new-output "std::unique_ptr<Edge>" :initarg nil :dont-save t)))) (edge-new-output "std::unique_ptr<Edge>" :initarg nil :dont-save t))))
(lcp:define-rpc vertex-count (lcp:define-rpc vertex-count
(:request ((member "tx::TransactionId" :capnp-type "UInt64"))) (:request ((member "tx::TransactionId")))
(:response ((member :int64_t)))) (:response ((member :int64_t))))
(lcp:pop-namespace) ;; distributed (lcp:pop-namespace) ;; distributed

9
src/distributed/durability_rpc_messages.lcp
View File

@ -2,7 +2,6 @@
#pragma once #pragma once
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/durability_rpc_messages.capnp.h"
#include "durability/distributed/recovery.hpp" #include "durability/distributed/recovery.hpp"
#include "durability/distributed/serialization.hpp" #include "durability/distributed/serialization.hpp"
#include "transactions/transaction.hpp" #include "transactions/transaction.hpp"
@ -10,16 +9,12 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-import 'dur "/durability/distributed/serialization.capnp")
(lcp:capnp-namespace "distributed")
(lcp:define-rpc make-snapshot (lcp:define-rpc make-snapshot
(:request ((member "tx::TransactionId" :capnp-type "UInt64"))) (:request ((member "tx::TransactionId")))
(:response ((member :bool)))) (:response ((member :bool))))
(lcp:define-rpc recover-wal-and-indexes (lcp:define-rpc recover-wal-and-indexes
(:request ((member "durability::RecoveryData" :capnp-type "Dur.RecoveryData"))) (:request ((member "durability::RecoveryData")))
(:response ())) (:response ()))
(lcp:pop-namespace) ;; distributed (lcp:pop-namespace) ;; distributed

29
src/distributed/dynamic_worker_rpc_messages.lcp
View File

@ -5,41 +5,14 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/dynamic_worker_rpc_messages.capnp.h"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
cpp<# cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:define-rpc dynamic-worker (lcp:define-rpc dynamic-worker
(:request ()) (:request ())
(:response (:response
((recover-indices "std::vector<std::pair<std::string, std::string>>" ((recover-indices "std::vector<std::pair<std::string, std::string>>"))))
:capnp-type "List(Utils.Pair(Text, Text))"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveVector<utils::capnp::Pair<::capnp::Text, ::capnp::Text>,
std::pair<std::string, std::string>>(
${member}, &${builder}, [](auto *builder, const auto value) {
builder->setFirst(value.first);
builder->setSecond(value.second);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::LoadVector<utils::capnp::Pair<::capnp::Text, ::capnp::Text>,
std::pair<std::string, std::string>>(
&${member}, ${reader}, [](const auto &reader) {
return std::make_pair(reader.getFirst(), reader.getSecond());
});
cpp<#)))))
(lcp:pop-namespace) ;; distributed (lcp:pop-namespace) ;; distributed

15
src/distributed/index_rpc_messages.lcp
View File

@ -5,7 +5,6 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/index_rpc_messages.capnp.h"
#include "storage/common/types/types.hpp" #include "storage/common/types/types.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
#include "transactions/transaction.hpp" #include "transactions/transaction.hpp"
@ -13,21 +12,17 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:define-rpc populate-index (lcp:define-rpc populate-index
(:request (:request
((label "storage::Label" :capnp-type "Storage.Label") ((label "storage::Label")
(property "storage::Property" :capnp-type "Storage.Property") (property "storage::Property")
(tx-id "tx::TransactionId" :capnp-type "UInt64"))) (tx-id "tx::TransactionId")))
(:response ())) (:response ()))
(lcp:define-rpc create-index (lcp:define-rpc create-index
(:request (:request
((label "storage::Label" :capnp-type "Storage.Label") ((label "storage::Label")
(property "storage::Property" :capnp-type "Storage.Property"))) (property "storage::Property")))
(:response ())) (:response ()))
(lcp:pop-namespace) ;; distributed (lcp:pop-namespace) ;; distributed

38
src/distributed/plan_rpc_messages.lcp
View File

@ -5,31 +5,10 @@
#include "query/frontend/ast/ast.hpp" #include "query/frontend/ast/ast.hpp"
#include "query/frontend/semantic/symbol_table.hpp" #include "query/frontend/semantic/symbol_table.hpp"
#include "query/distributed/plan/ops.hpp" #include "query/distributed/plan/ops.hpp"
#include "distributed/plan_rpc_messages.capnp.h"
cpp<# cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:capnp-import 'plan "/query/distributed/plan/ops.capnp")
(lcp:capnp-import 'query "/query/distributed/serialization.capnp")
(defun load-plan (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
query::plan::LogicalOperator::LoadHelper helper;
${member} = utils::LoadSharedPtr<query::plan::capnp::LogicalOperator, query::plan::LogicalOperator>(
${reader}, [&helper](const auto &reader) {
std::unique_ptr<query::plan::LogicalOperator> op;
query::plan::Load(&op, reader, &helper);
return op.release();
}, &helper.loaded_ops);
self->storage = std::move(helper.ast_storage);
cpp<#)
(defun slk-save-plan (member) (defun slk-save-plan (member)
#>cpp #>cpp
query::plan::LogicalOperator::SaveHelper helper; query::plan::LogicalOperator::SaveHelper helper;
@ -50,26 +29,13 @@ cpp<#
self->storage = std::move(helper.ast_storage); self->storage = std::move(helper.ast_storage);
cpp<#) cpp<#)
(defun save-plan (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
query::plan::LogicalOperator::SaveHelper helper;
utils::SaveSharedPtr<query::plan::capnp::LogicalOperator, query::plan::LogicalOperator>(
${member}, &${builder},
[&helper](auto *builder, const auto &val) {
Save(val, builder, &helper);
}, &helper.saved_ops);
cpp<#)
(lcp:define-rpc dispatch-plan (lcp:define-rpc dispatch-plan
(:request (:request
((plan-id :int64_t) ((plan-id :int64_t)
(plan "std::shared_ptr<query::plan::LogicalOperator>" (plan "std::shared_ptr<query::plan::LogicalOperator>"
:capnp-type "Utils.SharedPtr(Plan.LogicalOperator)"
:slk-save #'slk-save-plan :slk-save #'slk-save-plan
:slk-load #'slk-load-plan :slk-load #'slk-load-plan)
:capnp-save #'save-plan :capnp-load #'load-plan) (symbol-table "query::SymbolTable")
(symbol-table "query::SymbolTable" :capnp-type "Query.SymbolTable")
(storage "query::AstStorage" :initarg nil :dont-save t))) (storage "query::AstStorage" :initarg nil :dont-save t)))
(:response ())) (:response ()))

73
src/distributed/pull_produce_rpc_messages.lcp
View File

@ -6,14 +6,12 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/pull_produce_rpc_messages.capnp.h"
#include "query/context.hpp" #include "query/context.hpp"
#include "query/frontend/semantic/symbol.hpp" #include "query/frontend/semantic/symbol.hpp"
#include "query/parameters.hpp" #include "query/parameters.hpp"
#include "query/distributed/serialization.hpp" #include "query/distributed/serialization.hpp"
#include "storage/distributed/address_types.hpp" #include "storage/distributed/address_types.hpp"
#include "transactions/type.hpp" #include "transactions/type.hpp"
#include "rpc/serialization.hpp"
cpp<# cpp<#
(lcp:in-impl (lcp:in-impl
@ -27,16 +25,6 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:capnp-import 'query "/query/distributed/serialization.capnp")
(lcp:capnp-import 'sem "/query/distributed/frontend/semantic/symbol_serialization.capnp")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:capnp-type-conversion "tx::CommandId" "UInt32")
(lcp:capnp-type-conversion "tx::TransactionId" "UInt64")
#>cpp #>cpp
// Forward declare for LoadGraphElement. // Forward declare for LoadGraphElement.
class DataManager; class DataManager;
@ -100,38 +88,7 @@ the relevant parts of the response, ready for use."))
((pull-state "PullState") ((pull-state "PullState")
(frames "std::vector<std::vector<query::TypedValue>>" (frames "std::vector<std::vector<query::TypedValue>>"
:slk-save #'slk-save-frames :slk-save #'slk-save-frames
:slk-load #'slk-load-frames :slk-load #'slk-load-frames)
:capnp-type "List(List(Query.TypedValue))"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
for (size_t frame_i = 0; frame_i < ${member}.size(); ++frame_i) {
const auto &frame = ${member}[frame_i];
auto frame_builder = ${builder}.init(frame_i, frame.size());
for (size_t val_i = 0; val_i < frame.size(); ++val_i) {
const auto &value = frame[val_i];
auto value_builder = frame_builder[val_i];
query::SaveCapnpTypedValue(value, &value_builder, self.send_versions, self.worker_id);
}
}
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
${member}.reserve(${reader}.size());
for (const auto &frame_reader : ${reader}) {
std::vector<query::TypedValue> current_frame;
current_frame.reserve(frame_reader.size());
for (const auto &value_reader : frame_reader) {
query::TypedValue value;
query::LoadCapnpTypedValue(value_reader, &value, dba, data_manager);
current_frame.emplace_back(value);
}
${member}.emplace_back(current_frame);
}
cpp<#))
(worker-id :int16_t :dont-save t (worker-id :int16_t :dont-save t
:documentation :documentation
"Id of the worker on which the response is created, used for "Id of the worker on which the response is created, used for
@ -206,37 +163,23 @@ to the appropriate value. Not used on side that generates the response.")
PullResData &operator=(PullResData &&) = default; PullResData &operator=(PullResData &&) = default;
cpp<#) cpp<#)
(:serialize (:slk :load-args '((dba "database::GraphDbAccessor *") (:serialize (:slk :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))) (data-manager "distributed::DataManager *")))))
(:capnp :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *")))))
(lcp:define-rpc pull (lcp:define-rpc pull
(:request (:request
((tx-id "tx::TransactionId") ((tx-id "tx::TransactionId")
(tx-snapshot "tx::Snapshot" (tx-snapshot "tx::Snapshot"
:slk-save #'slk-save-snapshot :slk-save #'slk-save-snapshot
:slk-load #'slk-load-snapshot :slk-load #'slk-load-snapshot)
:capnp-type "List(UInt64)"
:capnp-init nil
:capnp-save #'save-snapshot
:capnp-load #'load-snapshot)
(plan-id :int64_t) (plan-id :int64_t)
(command-id "tx::CommandId") (command-id "tx::CommandId")
(timestamp :int64_t) (timestamp :int64_t)
(parameters "query::Parameters" (parameters "query::Parameters")
:capnp-type "Utils.Map(Utils.BoxInt64, Storage.PropertyValue)") (symbols "std::vector<query::Symbol>")
(symbols "std::vector<query::Symbol>" :capnp-type "List(Sem.Symbol)")
(accumulate :bool) (accumulate :bool)
(batch-size :int64_t) (batch-size :int64_t)
;; Indicates which of (old, new) records of a graph element should be sent. ;; Indicates which of (old, new) records of a graph element should be sent.
(send-versions "storage::SendVersions" (send-versions "storage::SendVersions")))
:capnp-type "Storage.SendVersions" :capnp-init nil
:capnp-save (lcp:capnp-save-enum "storage::capnp::SendVersions"
"storage::SendVersions"
'(both only-old only-new))
:capnp-load (lcp:capnp-load-enum "storage::capnp::SendVersions"
"storage::SendVersions"
'(both only-old only-new)))))
(:response (:response
((data "PullResData" :initarg :move ((data "PullResData" :initarg :move
:slk-load (lambda (m) :slk-load (lambda (m)
@ -244,9 +187,7 @@ to the appropriate value. Not used on side that generates the response.")
slk::Load(&self->${m}, reader, dba, data_manager); slk::Load(&self->${m}, reader, dba, data_manager);
cpp<#))) cpp<#)))
(:serialize (:slk :load-args '((dba "database::GraphDbAccessor *") (:serialize (:slk :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))) (data-manager "distributed::DataManager *"))))))
(:capnp :load-args '((dba "database::GraphDbAccessor *")
(data-manager "distributed::DataManager *"))))))
;; TODO make a separate RPC for the continuation of an existing pull, as an ;; TODO make a separate RPC for the continuation of an existing pull, as an
;; optimization not to have to send the full PullReqData pack every time. ;; optimization not to have to send the full PullReqData pack every time.

5
src/distributed/storage_gc_rpc_messages.lcp
View File

@ -2,7 +2,6 @@
#pragma once #pragma once
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/storage_gc_rpc_messages.capnp.h"
#include "io/network/endpoint.hpp" #include "io/network/endpoint.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
#include "transactions/transaction.hpp" #include "transactions/transaction.hpp"
@ -10,11 +9,9 @@ cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:define-rpc ran-local-gc (lcp:define-rpc ran-local-gc
(:request (:request
((local-oldest-active "tx::TransactionId" :capnp-type "UInt64") ((local-oldest-active "tx::TransactionId")
(worker-id :int16_t))) (worker-id :int16_t)))
(:response ())) (:response ()))

3
src/distributed/token_sharing_rpc_messages.lcp
View File

@ -5,14 +5,11 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "distributed/token_sharing_rpc_messages.capnp.h"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
cpp<# cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:define-rpc token-transfer (lcp:define-rpc token-transfer
(:request ()) (:request ())
(:response ())) (:response ()))

108
src/distributed/updates_rpc_messages.lcp
View File

@ -5,31 +5,15 @@
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "database/distributed/serialization.hpp" #include "database/distributed/serialization.hpp"
#include "distributed/updates_rpc_messages.capnp.h"
#include "durability/distributed/state_delta.hpp" #include "durability/distributed/state_delta.hpp"
#include "storage/distributed/address_types.hpp" #include "storage/distributed/address_types.hpp"
#include "storage/distributed/gid.hpp" #include "storage/distributed/gid.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
#include "transactions/type.hpp" #include "transactions/type.hpp"
#include "rpc/serialization.hpp"
cpp<# cpp<#
(lcp:namespace distributed) (lcp:namespace distributed)
(lcp:capnp-namespace "distributed")
(lcp:capnp-import 'db "/database/distributed/serialization.capnp")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:capnp-type-conversion "tx::TransactionId" "UInt64")
(lcp:capnp-type-conversion "gid::Gid" "UInt64")
(lcp:capnp-type-conversion "storage::Label" "Storage.Label")
(lcp:capnp-type-conversion "storage::EdgeType" "Storage.EdgeType")
(lcp:capnp-type-conversion "storage::Property" "Storage.Property")
(lcp:capnp-type-conversion "storage::EdgeAddress" "Storage.Address")
(lcp:capnp-type-conversion "storage::VertexAddress" "Storage.Address")
(lcp:define-enum update-result (lcp:define-enum update-result
(done (done
serialization-error serialization-error
@ -40,7 +24,7 @@ cpp<#
(:serialize)) (:serialize))
(lcp:define-rpc update (lcp:define-rpc update
(:request ((member "database::StateDelta" :capnp-type "Db.StateDelta") (:request ((member "database::StateDelta")
(worker-id :int64_t))) (worker-id :int64_t)))
(:response ((member "UpdateResult")))) (:response ((member "UpdateResult"))))
@ -52,63 +36,14 @@ cpp<#
((result "UpdateResult") ((result "UpdateResult")
(cypher-id :int64_t :documentation "Only valid if creation was successful.") (cypher-id :int64_t :documentation "Only valid if creation was successful.")
(gid "gid::Gid" :documentation "Only valid if creation was successful.")) (gid "gid::Gid" :documentation "Only valid if creation was successful."))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-struct create-vertex-req-data () (lcp:define-struct create-vertex-req-data ()
((tx-id "tx::TransactionId") ((tx-id "tx::TransactionId")
(labels "std::vector<storage::Label>" (labels "std::vector<storage::Label>")
:capnp-save (lcp:capnp-save-vector "storage::capnp::Label" "storage::Label") (properties "std::unordered_map<storage::Property, PropertyValue>")
:capnp-load (lcp:capnp-load-vector "storage::capnp::Label" "storage::Label")) (cypher-id "std::optional<int64_t>"))
(properties "std::unordered_map<storage::Property, PropertyValue>" (:serialize (:slk)))
:capnp-type "Utils.Map(Storage.Property, Storage.PropertyValue)"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveMap<storage::capnp::Property, storage::capnp::PropertyValue>(
${member}, &${builder},
[](auto *builder, const auto &entry) {
auto key_builder = builder->initKey();
Save(entry.first, &key_builder);
auto value_builder = builder->initValue();
storage::SaveCapnpPropertyValue(entry.second, &value_builder);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::LoadMap<storage::capnp::Property, storage::capnp::PropertyValue>(
&${member}, ${reader},
[](const auto &reader) {
storage::Property prop;
storage::Load(&prop, reader.getKey());
PropertyValue value;
storage::LoadCapnpPropertyValue(reader.getValue(), &value);
return std::make_pair(prop, value);
});
cpp<#))
(cypher-id "std::optional<int64_t>"
:capnp-type "Utils.Optional(Utils.BoxInt64)"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveOptional<utils::capnp::BoxInt64, int64_t>(
${member}, &${builder}, [](auto *builder, const auto &value) {
builder->setValue(value);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
${member} = utils::LoadOptional<utils::capnp::BoxInt64, int64_t>(
${reader}, [](const auto &reader) {
return reader.getValue();
});
cpp<#)))
(:serialize (:slk) (:capnp)))
(lcp:define-rpc create-vertex (lcp:define-rpc create-vertex
(:request ((member "CreateVertexReqData"))) (:request ((member "CreateVertexReqData")))
@ -120,27 +55,8 @@ cpp<#
(to "storage::VertexAddress") (to "storage::VertexAddress")
(edge-type "storage::EdgeType") (edge-type "storage::EdgeType")
(tx-id "tx::TransactionId") (tx-id "tx::TransactionId")
(cypher-id "std::optional<int64_t>" (cypher-id "std::optional<int64_t>"))
:capnp-type "Utils.Optional(Utils.BoxInt64)" (:serialize (:slk)))
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
utils::SaveOptional<utils::capnp::BoxInt64, int64_t>(
${member}, &${builder}, [](auto *builder, const auto &value) {
builder->setValue(value);
});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
${member} = utils::LoadOptional<utils::capnp::BoxInt64, int64_t>(
${reader}, [](const auto &reader) {
return reader.getValue();
});
cpp<#)))
(:serialize (:slk) (:capnp)))
(lcp:define-rpc create-edge (lcp:define-rpc create-edge
(:request ((member "CreateEdgeReqData"))) (:request ((member "CreateEdgeReqData")))
@ -153,7 +69,7 @@ cpp<#
(to "gid::Gid") (to "gid::Gid")
(edge-type "storage::EdgeType") (edge-type "storage::EdgeType")
(tx-id "tx::TransactionId")) (tx-id "tx::TransactionId"))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-rpc add-in-edge (lcp:define-rpc add-in-edge
(:request ((member "AddInEdgeReqData"))) (:request ((member "AddInEdgeReqData")))
@ -164,7 +80,7 @@ cpp<#
(gid "gid::Gid") (gid "gid::Gid")
(tx-id "tx::TransactionId") (tx-id "tx::TransactionId")
(check-empty :bool)) (check-empty :bool))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-rpc remove-vertex (lcp:define-rpc remove-vertex
(:request ((member "RemoveVertexReqData"))) (:request ((member "RemoveVertexReqData")))
@ -176,7 +92,7 @@ cpp<#
(edge-id "gid::Gid") (edge-id "gid::Gid")
(vertex-from-id "gid::Gid") (vertex-from-id "gid::Gid")
(vertex-to-address "storage::VertexAddress")) (vertex-to-address "storage::VertexAddress"))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-rpc remove-edge (lcp:define-rpc remove-edge
(:request ((member "RemoveEdgeData"))) (:request ((member "RemoveEdgeData")))
@ -187,7 +103,7 @@ cpp<#
(tx-id "tx::TransactionId") (tx-id "tx::TransactionId")
(vertex "gid::Gid") (vertex "gid::Gid")
(edge-address "storage::EdgeAddress")) (edge-address "storage::EdgeAddress"))
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:define-rpc remove-in-edge (lcp:define-rpc remove-in-edge
(:request ((member "RemoveInEdgeData"))) (:request ((member "RemoveInEdgeData")))

19
src/durability/distributed/serialization.capnp
View File

@ -1,19 +0,0 @@
@0xb3d70bc0576218f3;
using Cxx = import "/capnp/c++.capnp";
using Utils = import "/rpc/serialization.capnp";
$Cxx.namespace("durability::capnp");
struct RecoveryInfo {
durabilityVersion @0 :UInt64;
snapshotTxId @1 :UInt64;
walRecovered @2 :List(UInt64);
}
struct RecoveryData {
snapshooterTxId @0 :UInt64;
walTxToRecover @1 :List(UInt64);
snapshooterTxSnapshot @2 :List(UInt64);
indexes @3 :List(Utils.Pair(Text, Text));
}

67
src/durability/distributed/serialization.hpp
View File

@ -1,75 +1,8 @@
#pragma once #pragma once
#include "durability/distributed/recovery.hpp" #include "durability/distributed/recovery.hpp"
#include "durability/distributed/serialization.capnp.h"
#include "rpc/serialization.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
namespace durability {
inline void Save(const RecoveryInfo &info,
capnp::RecoveryInfo::Builder *builder) {
builder->setDurabilityVersion(info.durability_version);
builder->setSnapshotTxId(info.snapshot_tx_id);
auto list_builder = builder->initWalRecovered(info.wal_recovered.size());
utils::SaveVector(info.wal_recovered, &list_builder);
}
inline void Load(RecoveryInfo *info,
const capnp::RecoveryInfo::Reader &reader) {
info->durability_version = reader.getDurabilityVersion();
info->snapshot_tx_id = reader.getSnapshotTxId();
auto list_reader = reader.getWalRecovered();
utils::LoadVector(&info->wal_recovered, list_reader);
}
inline void Save(const RecoveryData &data,
capnp::RecoveryData::Builder *builder) {
builder->setSnapshooterTxId(data.snapshooter_tx_id);
{
auto list_builder =
builder->initWalTxToRecover(data.wal_tx_to_recover.size());
utils::SaveVector(data.wal_tx_to_recover, &list_builder);
}
{
auto list_builder =
builder->initSnapshooterTxSnapshot(data.snapshooter_tx_snapshot.size());
utils::SaveVector(data.snapshooter_tx_snapshot, &list_builder);
}
{
auto list_builder = builder->initIndexes(data.indexes.size());
utils::SaveVector<utils::capnp::Pair<::capnp::Text, ::capnp::Text>,
std::pair<std::string, std::string>>(
data.indexes, &list_builder, [](auto *builder, const auto value) {
builder->setFirst(value.first);
builder->setSecond(value.second);
});
}
}
inline void Load(RecoveryData *data,
const capnp::RecoveryData::Reader &reader) {
data->snapshooter_tx_id = reader.getSnapshooterTxId();
{
auto list_reader = reader.getWalTxToRecover();
utils::LoadVector(&data->wal_tx_to_recover, list_reader);
}
{
auto list_reader = reader.getSnapshooterTxSnapshot();
utils::LoadVector(&data->snapshooter_tx_snapshot, list_reader);
}
{
auto list_reader = reader.getIndexes();
utils::LoadVector<utils::capnp::Pair<::capnp::Text, ::capnp::Text>,
std::pair<std::string, std::string>>(
&data->indexes, list_reader, [](const auto &reader) {
return std::make_pair(reader.getFirst(), reader.getSecond());
});
}
}
} // namespace durability
namespace slk { namespace slk {
inline void Save(const durability::RecoveryInfo &info, slk::Builder *builder) { inline void Save(const durability::RecoveryInfo &info, slk::Builder *builder) {

29
src/durability/distributed/state_delta.lcp
View File

@ -18,18 +18,6 @@ cpp<#
class GraphDbAccessor; class GraphDbAccessor;
cpp<# cpp<#
(lcp:capnp-namespace "database")
(lcp:capnp-import 'storage "/storage/distributed/rpc/serialization.capnp")
(lcp:capnp-type-conversion "tx::TransactionId" "UInt64")
(lcp:capnp-type-conversion "gid::Gid" "UInt64")
(lcp:capnp-type-conversion "storage::Label" "Storage.Label")
(lcp:capnp-type-conversion "storage::EdgeType" "Storage.EdgeType")
(lcp:capnp-type-conversion "storage::Property" "Storage.Property")
(lcp:capnp-type-conversion "storage::EdgeAddress" "Storage.Address")
(lcp:capnp-type-conversion "storage::VertexAddress" "Storage.Address")
(lcp:define-struct state-delta () (lcp:define-struct state-delta ()
( (
;; Members valid for every delta. ;; Members valid for every delta.
@ -50,20 +38,7 @@ cpp<#
(edge-type-name "std::string") (edge-type-name "std::string")
(property "storage::Property") (property "storage::Property")
(property-name "std::string") (property-name "std::string")
(value "PropertyValue" :initval "PropertyValue::Null" (value "PropertyValue" :initval "PropertyValue::Null")
:capnp-type "Storage.PropertyValue"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
storage::SaveCapnpPropertyValue(${member}, &${builder});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
storage::LoadCapnpPropertyValue(${reader}, &${member});
cpp<#))
(label "storage::Label") (label "storage::Label")
(label-name "std::string") (label-name "std::string")
(check-empty :bool)) (check-empty :bool))
@ -171,6 +146,6 @@ omitted in the comment.")
/// Applies CRUD delta to database accessor. Fails on other types of deltas /// Applies CRUD delta to database accessor. Fails on other types of deltas
void Apply(GraphDbAccessor &dba) const; void Apply(GraphDbAccessor &dba) const;
cpp<#) cpp<#)
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:pop-namespace) ;; database (lcp:pop-namespace) ;; database

27
src/durability/single_node_ha/state_delta.lcp
View File

@ -21,16 +21,6 @@ cpp<#
class GraphDbAccessor; class GraphDbAccessor;
cpp<# cpp<#
(lcp:capnp-namespace "database")
(lcp:capnp-import 'storage "/storage/single_node_ha/rpc/serialization.capnp")
(lcp:capnp-type-conversion "tx::TransactionId" "UInt64")
(lcp:capnp-type-conversion "gid::Gid" "UInt64")
(lcp:capnp-type-conversion "storage::Label" "Storage.Label")
(lcp:capnp-type-conversion "storage::EdgeType" "Storage.EdgeType")
(lcp:capnp-type-conversion "storage::Property" "Storage.Property")
(lcp:define-struct state-delta () (lcp:define-struct state-delta ()
( (
;; Members valid for every delta. ;; Members valid for every delta.
@ -45,20 +35,7 @@ cpp<#
(edge-type-name "std::string") (edge-type-name "std::string")
(property "storage::Property") (property "storage::Property")
(property-name "std::string") (property-name "std::string")
(value "PropertyValue" :initval "PropertyValue::Null" (value "PropertyValue" :initval "PropertyValue::Null")
:capnp-type "Storage.PropertyValue"
:capnp-save
(lambda (builder member capnp-name)
(declare (ignore capnp-name))
#>cpp
storage::SaveCapnpPropertyValue(${member}, &${builder});
cpp<#)
:capnp-load
(lambda (reader member capnp-name)
(declare (ignore capnp-name))
#>cpp
storage::LoadCapnpPropertyValue(${reader}, &${member});
cpp<#))
(label "storage::Label") (label "storage::Label")
(label-name "std::string") (label-name "std::string")
(check-empty :bool) (check-empty :bool)
@ -156,6 +133,6 @@ omitted in the comment.")
/// Applies CRUD delta to database accessor. Fails on other types of deltas /// Applies CRUD delta to database accessor. Fails on other types of deltas
void Apply(GraphDbAccessor &dba) const; void Apply(GraphDbAccessor &dba) const;
cpp<#) cpp<#)
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:pop-namespace) ;; database (lcp:pop-namespace) ;; database

10
src/io/CMakeLists.txt
View File

@ -6,13 +6,3 @@ set(io_src_files
add_library(mg-io STATIC ${io_src_files}) add_library(mg-io STATIC ${io_src_files})
target_link_libraries(mg-io stdc++fs Threads::Threads fmt glog mg-utils) target_link_libraries(mg-io stdc++fs Threads::Threads fmt glog mg-utils)
set(io_serialization_src_files)
define_add_capnp(add_capnp io_serialization_src_files io_capnp_files)
add_capnp(network/endpoint.capnp)
add_custom_target(generate_io_capnp DEPENDS ${io_capnp_files})
add_library(mg-io-serialization STATIC ${io_serialization_src_files})
target_link_libraries(mg-io-serialization mg-io capnp kj)
add_dependencies(mg-io-serialization generate_io_capnp)

10
src/io/network/endpoint.capnp
View File

@ -1,10 +0,0 @@
@0x93c2449a1e02365a;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("io::network::capnp");
struct Endpoint {
address @0 :Text;
port @1 :UInt16;
family @2 :UInt8;
}

17
src/io/network/serialization.hpp
View File

@ -1,25 +1,8 @@
#pragma once #pragma once
#include "io/network/endpoint.capnp.h"
#include "io/network/endpoint.hpp" #include "io/network/endpoint.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
namespace io::network {
inline void Save(const Endpoint &endpoint, capnp::Endpoint::Builder *builder) {
builder->setAddress(endpoint.address());
builder->setPort(endpoint.port());
builder->setFamily(endpoint.family());
}
inline void Load(Endpoint *endpoint, const capnp::Endpoint::Reader &reader) {
endpoint->address_ = reader.getAddress();
endpoint->port_ = reader.getPort();
endpoint->family_ = reader.getFamily();
}
} // namespace io::network
namespace slk { namespace slk {
inline void Save(const io::network::Endpoint &endpoint, slk::Builder *builder) { inline void Save(const io::network::Endpoint &endpoint, slk::Builder *builder) {

27
src/lisp/CMakeLists.txt
View File

@ -23,24 +23,16 @@ add_custom_target(lcp
# The `define_add_lcp` expects 3 arguments: # The `define_add_lcp` expects 3 arguments:
# * name -- name for the function, you usually want `add_lcp` # * name -- name for the function, you usually want `add_lcp`
# * main_src_files -- variable to be updated with generated cpp files # * main_src_files -- variable to be updated with generated cpp files
# * generated_lcp_files -- variable to be updated with generated hpp, cpp and capnp files # * generated_lcp_files -- variable to be updated with generated hpp and cpp files
# #
# The `add_lcp` function expects at least a single argument, path to lcp file. # The `add_lcp` function expects at least a single argument, path to lcp file.
# Each added file is standalone and we avoid recompiling everything. # Each added file is standalone and we avoid recompiling everything.
# #
# By default, each `.lcp` file will produce a `.hpp` and `.cpp` file. To tell # By default, each `.lcp` file will produce a `.hpp` and `.cpp` file. To tell
# CMake that no `.cpp` file will be generated, pass a NO_CPP option. # CMake that no `.cpp` file will be generated, pass a NO_CPP option.
#
# You may pass a CAPNP_SCHEMA <id> keyword argument to generate the Cap'n Proto
# serialization code from .lcp file. You still need to add the generated capnp
# file through `add_capnp` function. To generate the <id> use `capnp id`
# invocation, and specify it here. This preserves correct id information across
# multiple schema generations. If this wasn't the case, wrong typeId
# information will break serialization between different compilations.
macro(define_add_lcp name main_src_files generated_lcp_files) macro(define_add_lcp name main_src_files generated_lcp_files)
function(${name} lcp_file) function(${name} lcp_file)
set(options NO_CPP) set(options NO_CPP SLK_SERIALIZE)
set(one_value_kwargs CAPNP_SCHEMA)
set(multi_value_kwargs DEPENDS) set(multi_value_kwargs DEPENDS)
# NOTE: ${${}ARGN} syntax escapes evaluating macro's ARGN variable; see: # NOTE: ${${}ARGN} syntax escapes evaluating macro's ARGN variable; see:
# https://stackoverflow.com/questions/50365544/how-to-access-enclosing-functions-arguments-from-within-a-macro # https://stackoverflow.com/questions/50365544/how-to-access-enclosing-functions-arguments-from-within-a-macro
@ -52,11 +44,8 @@ macro(define_add_lcp name main_src_files generated_lcp_files)
# Update *global* main_src_files # Update *global* main_src_files
set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE) set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
endif() endif()
if (KW_CAPNP_SCHEMA) if (KW_SLK_SERIALIZE)
string(REGEX REPLACE "\.lcp$" ".capnp" capnp_file set(slk_serialize "SLK_SERIALIZE")
"${CMAKE_CURRENT_SOURCE_DIR}/${lcp_file}")
set(capnp_id ${KW_CAPNP_SCHEMA})
set(capnp_depend capnproto-proj)
endif() endif()
# Repeat the `lcp_src_files` because this is a macro and the variable is # Repeat the `lcp_src_files` because this is a macro and the variable is
# not visible when invoked in another file. # not visible when invoked in another file.
@ -71,12 +60,12 @@ macro(define_add_lcp name main_src_files generated_lcp_files)
${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp ${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp
${CMAKE_SOURCE_DIR}/src/lisp/lcp-test.lisp ${CMAKE_SOURCE_DIR}/src/lisp/lcp-test.lisp
${CMAKE_SOURCE_DIR}/tools/lcp) ${CMAKE_SOURCE_DIR}/tools/lcp)
add_custom_command(OUTPUT ${h_file} ${cpp_file} ${capnp_file} add_custom_command(OUTPUT ${h_file} ${cpp_file}
COMMAND ${CMAKE_SOURCE_DIR}/tools/lcp ${lcp_file} ${capnp_id} COMMAND ${CMAKE_SOURCE_DIR}/tools/lcp ${lcp_file} ${slk_serialize}
VERBATIM VERBATIM
DEPENDS ${lcp_src_files} lcp ${lcp_file} ${capnp_depend} ${KW_DEPENDS} DEPENDS ${lcp_src_files} lcp ${lcp_file}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
# Update *global* generated_lcp_files # Update *global* generated_lcp_files
set(${generated_lcp_files} ${${generated_lcp_files}} ${h_file} ${cpp_file} ${capnp_file} PARENT_SCOPE) set(${generated_lcp_files} ${${generated_lcp_files}} ${h_file} ${cpp_file} PARENT_SCOPE)
endfunction(${name}) endfunction(${name})
endmacro(define_add_lcp) endmacro(define_add_lcp)

9
src/lisp/lcp.lisp
View File

@ -1324,10 +1324,7 @@ enums which aren't defined in LCP."
(declare (type list request response)) (declare (type list request response))
(assert (eq :request (car request))) (assert (eq :request (car request)))
(assert (eq :response (car response))) (assert (eq :response (car response)))
(flet ((decl-type-info (class-name) (flet ((decl-type-info (class-name))
#>cpp
using Capnp = capnp::${class-name};
cpp<#)
(def-constructor (class-name members) (def-constructor (class-name members)
(let ((full-constructor (let ((full-constructor
(let ((init-members (remove-if (lambda (slot-def) (let ((init-members (remove-if (lambda (slot-def)
@ -1496,7 +1493,7 @@ code generation."
(write-line (cpp-enum-to-capnp-function-definition cpp-type) cpp-out) (write-line (cpp-enum-to-capnp-function-definition cpp-type) cpp-out)
(write-line (cpp-enum-from-capnp-function-definition cpp-type) cpp-out)))))) (write-line (cpp-enum-from-capnp-function-definition cpp-type) cpp-out))))))
(defun process-file (lcp-file &key capnp-id) (defun process-file (lcp-file &key capnp-id slk-serialize)
"Process a LCP-FILE and write the output to .hpp file in the same directory. "Process a LCP-FILE and write the output to .hpp file in the same directory.
If CAPNP-ID is passed, generates the Cap'n Proto schema to .capnp file in the If CAPNP-ID is passed, generates the Cap'n Proto schema to .capnp file in the
same directory, while the loading code is generated in LCP-FILE.cpp source same directory, while the loading code is generated in LCP-FILE.cpp source
@ -1509,7 +1506,7 @@ file."
;; have our own accompanying .cpp files ;; have our own accompanying .cpp files
(cpp-file (concatenate 'string lcp-file ".cpp")) (cpp-file (concatenate 'string lcp-file ".cpp"))
(capnp-file (concatenate 'string filename ".capnp")) (capnp-file (concatenate 'string filename ".capnp"))
(serializep capnp-id) (serializep slk-serialize)
;; Reset globals ;; Reset globals
(*capnp-namespace* nil) (*capnp-namespace* nil)
(*capnp-imports* nil) (*capnp-imports* nil)

10
src/query/distributed/frontend/ast/ast_serialization.lcp
View File

@ -4,7 +4,6 @@
#include <type_traits> #include <type_traits>
#include "query/frontend/ast/ast.hpp" #include "query/frontend/ast/ast.hpp"
#include "query/distributed/frontend/ast/ast_serialization.capnp.h"
#include "query/distributed/serialization.hpp" #include "query/distributed/serialization.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
cpp<# cpp<#
@ -17,8 +16,6 @@ cpp<#
/// Primary function for saving Ast nodes via SLK. /// Primary function for saving Ast nodes via SLK.
void SaveAstPointer(const Tree *ast, slk::Builder *builder); void SaveAstPointer(const Tree *ast, slk::Builder *builder);
Tree *Load(AstStorage *ast, const capnp::Tree::Reader &tree);
Tree *Load(AstStorage *ast, slk::Reader *reader); Tree *Load(AstStorage *ast, slk::Reader *reader);
/// Primary function for loading Ast nodes via SLK. /// Primary function for loading Ast nodes via SLK.
@ -54,13 +51,6 @@ cpp<#
ast->storage_.emplace_back(std::move(root)); ast->storage_.emplace_back(std::move(root));
return ast->storage_.back().get(); return ast->storage_.back().get();
} }
Tree *Load(AstStorage *ast, const capnp::Tree::Reader &tree) {
std::unique_ptr<Tree> root;
::query::Load(&root, tree, ast);
ast->storage_.emplace_back(std::move(root));
return ast->storage_.back().get();
}
cpp<#) cpp<#)
(lcp:pop-namespace) ;; namespace query (lcp:pop-namespace) ;; namespace query

1
src/query/distributed/frontend/semantic/symbol_serialization.lcp
View File

@ -2,7 +2,6 @@
#pragma once #pragma once
#include "query/frontend/semantic/symbol.hpp" #include "query/frontend/semantic/symbol.hpp"
#include "query/distributed/frontend/semantic/symbol_serialization.capnp.h"
cpp<# cpp<#
;; Generate serialization code ;; Generate serialization code

97
src/query/distributed/plan/ops.lcp
View File

@ -4,7 +4,6 @@
#pragma once #pragma once
#include "query/distributed/frontend/ast/ast_serialization.hpp" #include "query/distributed/frontend/ast/ast_serialization.hpp"
#include "query/distributed/plan/ops.capnp.h"
#include "query/plan/operator.hpp" #include "query/plan/operator.hpp"
#include "query/distributed/serialization.hpp" #include "query/distributed/serialization.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
@ -52,13 +51,9 @@ cpp<#
(lcp:define-class pull-remote (logical-operator) (lcp:define-class pull-remote (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(plan-id :int64_t :initval 0 :scope :public) (plan-id :int64_t :initval 0 :scope :public)
(symbols "std::vector<Symbol>" :scope :public (symbols "std::vector<Symbol>" :scope :public))
:capnp-save (lcp:capnp-save-vector "::query::capnp::Symbol" "Symbol")
:capnp-load (lcp:capnp-load-vector "::query::capnp::Symbol" "Symbol")))
(:documentation (:documentation
"An operator in distributed Memgraph that yields both local and remote (from "An operator in distributed Memgraph that yields both local and remote (from
other workers) frames. Obtaining remote frames is done through RPC calls to other workers) frames. Obtaining remote frames is done through RPC calls to
@ -84,7 +79,7 @@ time on data transfer. It gives no guarantees on result order.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(defun slk-load-pull-remote (member) (defun slk-load-pull-remote (member)
@ -98,29 +93,13 @@ time on data transfer. It gives no guarantees on result order.")
self->${member} = std::static_pointer_cast<query::plan::PullRemote>(op); self->${member} = std::static_pointer_cast<query::plan::PullRemote>(op);
cpp<#) cpp<#)
(defun load-pull-remote (reader member-name capnp-name)
(declare (ignore capnp-name))
#>cpp
${member-name} = std::static_pointer_cast<PullRemote>(
utils::LoadSharedPtr<capnp::LogicalOperator, LogicalOperator>(${reader},
[helper](const auto &reader) {
std::unique_ptr<LogicalOperator> op;
Load(&op, reader, helper);
return op.release();
}, &helper->loaded_ops));
cpp<#)
(lcp:define-class synchronize (logical-operator) (lcp:define-class synchronize (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(pull-remote "std::shared_ptr<PullRemote>" :scope :public (pull-remote "std::shared_ptr<PullRemote>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-pull-remote :slk-load #'slk-load-pull-remote
:capnp-save #'save-operator-pointer
:capnp-load #'load-pull-remote
:clone (lambda (source dest) :clone (lambda (source dest)
#>cpp #>cpp
if (${source}) { if (${source}) {
@ -174,27 +153,19 @@ Logic of the synchronize operator is:
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:define-class pull-remote-order-by (logical-operator) (lcp:define-class pull-remote-order-by (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(plan-id :int64_t :initval 0 :scope :public) (plan-id :int64_t :initval 0 :scope :public)
(symbols "std::vector<Symbol>" :scope :public (symbols "std::vector<Symbol>" :scope :public)
:capnp-save (lcp:capnp-save-vector "::query::capnp::Symbol" "Symbol")
:capnp-load (lcp:capnp-load-vector "::query::capnp::Symbol" "Symbol"))
(order-by "std::vector<Expression *>" :scope :public (order-by "std::vector<Expression *>" :scope :public
:slk-save #'slk-save-ast-vector :slk-save #'slk-save-ast-vector
:slk-load (slk-load-ast-vector "Expression") :slk-load (slk-load-ast-vector "Expression"))
:capnp-type "List(Ast.Tree)" (compare "TypedValueVectorCompare" :scope :public))
:capnp-save (save-ast-vector "Expression *")
:capnp-load (load-ast-vector "Expression *"))
(compare "TypedValueVectorCompare" :scope :public
:capnp-type "Query.TypedValueVectorCompare"))
(:documentation (:documentation
"Operator that merges distributed OrderBy operators. "Operator that merges distributed OrderBy operators.
Instead of using a regular OrderBy on master (which would collect all remote Instead of using a regular OrderBy on master (which would collect all remote
@ -220,25 +191,16 @@ by having only one result from each worker.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:define-class distributed-expand (logical-operator) (lcp:define-class distributed-expand (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(input-symbol "Symbol" :scope :public) (input-symbol "Symbol" :scope :public)
(common "ExpandCommon" :scope :public) (common "ExpandCommon" :scope :public)
(graph-view "GraphView" :scope :public (graph-view "GraphView" :scope :public
:capnp-init nil
:capnp-save (lcp:capnp-save-enum "::query::capnp::GraphView"
"GraphView"
'(old new))
:capnp-load (lcp:capnp-load-enum "::query::capnp::GraphView"
"GraphView"
'(old new))
:documentation :documentation
"State from which the input node should get expanded.")) "State from which the input node should get expanded."))
(:documentation "Distributed version of Expand operator") (:documentation "Distributed version of Expand operator")
@ -264,42 +226,29 @@ by having only one result from each worker.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:define-class distributed-expand-bfs (logical-operator) (lcp:define-class distributed-expand-bfs (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(input-symbol "Symbol" :scope :public) (input-symbol "Symbol" :scope :public)
(common "ExpandCommon" :scope :public) (common "ExpandCommon" :scope :public)
(lower-bound "Expression *" :scope :public (lower-bound "Expression *" :scope :public
:documentation "Optional lower bound, default is 1" :documentation "Optional lower bound, default is 1"
:slk-save #'slk-save-ast-pointer :slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression") :slk-load (slk-load-ast-pointer "Expression"))
:capnp-type "Ast.Tree" :capnp-init nil
:capnp-save #'save-ast-pointer
:capnp-load (load-ast-pointer "Expression *"))
(upper-bound "Expression *" :scope :public (upper-bound "Expression *" :scope :public
:documentation "Optional upper bound, default is infinity" :documentation "Optional upper bound, default is infinity"
:slk-save #'slk-save-ast-pointer :slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression") :slk-load (slk-load-ast-pointer "Expression"))
:capnp-type "Ast.Tree" :capnp-init nil
:capnp-save #'save-ast-pointer
:capnp-load (load-ast-pointer "Expression *"))
(filter-lambda "ExpansionLambda" :scope :public (filter-lambda "ExpansionLambda" :scope :public
:documentation "Filter that must be satisfied for expansion to succeed." :documentation "Filter that must be satisfied for expansion to succeed."
:slk-load (lambda (member) :slk-load (lambda (member)
#>cpp #>cpp
slk::Load(&self->${member}, reader, &helper->ast_storage); slk::Load(&self->${member}, reader, &helper->ast_storage);
cpp<#) cpp<#)))
:capnp-type "ExpansionLambda"
:capnp-load (lambda (reader member capnp-name)
#>cpp
Load(&${member}, ${reader}, &helper->ast_storage);
cpp<#)))
(:documentation "BFS expansion operator suited for distributed execution.") (:documentation "BFS expansion operator suited for distributed execution.")
(:public (:public
#>cpp #>cpp
@ -327,15 +276,13 @@ by having only one result from each worker.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:define-class distributed-create-node (logical-operator) (lcp:define-class distributed-create-node (logical-operator)
((input "std::shared_ptr<LogicalOperator>" :scope :public ((input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(node-info "NodeCreationInfo" :scope :public (node-info "NodeCreationInfo" :scope :public
:slk-save (lambda (m) :slk-save (lambda (m)
#>cpp #>cpp
@ -364,7 +311,7 @@ by having only one result from each worker.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:define-class distributed-create-expand (logical-operator) (lcp:define-class distributed-create-expand (logical-operator)
@ -388,9 +335,7 @@ by having only one result from each worker.")
cpp<#)) cpp<#))
(input "std::shared_ptr<LogicalOperator>" :scope :public (input "std::shared_ptr<LogicalOperator>" :scope :public
:slk-save #'slk-save-operator-pointer :slk-save #'slk-save-operator-pointer
:slk-load #'slk-load-operator-pointer :slk-load #'slk-load-operator-pointer)
:capnp-save #'save-operator-pointer
:capnp-load #'load-operator-pointer)
(input-symbol "Symbol" :scope :public) (input-symbol "Symbol" :scope :public)
(existing-node :bool :scope :public)) (existing-node :bool :scope :public))
(:documentation "Distributed version of CreateExpand") (:documentation "Distributed version of CreateExpand")
@ -412,7 +357,7 @@ by having only one result from each worker.")
input_ = input; input_ = input;
} }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp)) (:serialize (:slk))
(:clone)) (:clone))
(lcp:pop-namespace) ;; plan (lcp:pop-namespace) ;; plan

4
src/query/distributed/plan/planner.cpp
View File

@ -2,10 +2,6 @@
#include <memory> #include <memory>
// TODO: Remove these includes for hacked cloning of logical operators via
// serialization when proper cloning is added.
#include <capnp/message.h>
#include "query/distributed/plan/ops.hpp" #include "query/distributed/plan/ops.hpp"
#include "query/distributed/plan/pretty_print.hpp" #include "query/distributed/plan/pretty_print.hpp"
#include "query/plan/operator.hpp" #include "query/plan/operator.hpp"

47
src/query/distributed/serialization.capnp
View File

@ -1,47 +0,0 @@
@0xf47e119e21912f20;
using Ast = import "/query/distributed/frontend/ast/ast_serialization.capnp";
using Cxx = import "/capnp/c++.capnp";
using Sem = import "/query/distributed/frontend/semantic/symbol_serialization.capnp";
using Storage = import "/storage/distributed/rpc/serialization.capnp";
using Utils = import "/rpc/serialization.capnp";
$Cxx.namespace("query::capnp");
enum GraphView {
old @0;
new @1;
}
struct TypedValueVectorCompare {
ordering @0 :List(Ast.Ordering);
}
struct TypedValue {
union {
nullType @0 :Void;
bool @1 :Bool;
integer @2 :Int64;
double @3 :Float64;
string @4 :Text;
list @5 :List(TypedValue);
map @6 :List(Entry);
vertex @7 :Storage.VertexAccessor;
edge @8 :Storage.EdgeAccessor;
path @9 :Path;
}
struct Entry {
key @0 :Text;
value @1 :TypedValue;
}
struct Path {
vertices @0 :List(Storage.VertexAccessor);
edges @1 :List(Storage.EdgeAccessor);
}
}
struct SymbolTable {
table @0 :Utils.Map(Utils.BoxInt32, Sem.Symbol);
}

188
src/query/distributed/serialization.cpp
View File

@ -2,194 +2,6 @@
#include "distributed/data_manager.hpp" #include "distributed/data_manager.hpp"
#include "query/distributed/frontend/ast/ast_serialization.hpp" #include "query/distributed/frontend/ast/ast_serialization.hpp"
#include "rpc/serialization.hpp"
namespace query {
void SaveCapnpTypedValue(const TypedValue &value,
capnp::TypedValue::Builder *builder,
storage::SendVersions versions, int worker_id) {
switch (value.type()) {
case TypedValue::Type::Null:
builder->setNullType();
return;
case TypedValue::Type::Bool:
builder->setBool(value.Value<bool>());
return;
case TypedValue::Type::Int:
builder->setInteger(value.Value<int64_t>());
return;
case TypedValue::Type::Double:
builder->setDouble(value.Value<double>());
return;
case TypedValue::Type::String:
builder->setString(value.Value<std::string>());
return;
case TypedValue::Type::List: {
const auto &values = value.Value<std::vector<TypedValue>>();
auto list_builder = builder->initList(values.size());
for (size_t i = 0; i < values.size(); ++i) {
auto value_builder = list_builder[i];
SaveCapnpTypedValue(values[i], &value_builder, versions, worker_id);
}
return;
}
case TypedValue::Type::Map: {
const auto &map = value.Value<std::map<std::string, TypedValue>>();
auto map_builder = builder->initMap(map.size());
size_t i = 0;
for (const auto &kv : map) {
auto kv_builder = map_builder[i];
kv_builder.setKey(kv.first);
auto value_builder = kv_builder.initValue();
SaveCapnpTypedValue(kv.second, &value_builder, versions, worker_id);
++i;
}
return;
}
case TypedValue::Type::Vertex: {
auto vertex_builder = builder->initVertex();
storage::SaveVertexAccessor(value.ValueVertex(), &vertex_builder,
versions, worker_id);
return;
}
case TypedValue::Type::Edge: {
auto edge_builder = builder->initEdge();
storage::SaveEdgeAccessor(value.ValueEdge(), &edge_builder, versions,
worker_id);
return;
}
case TypedValue::Type::Path: {
auto path_builder = builder->initPath();
const auto &path = value.ValuePath();
auto vertices_builder = path_builder.initVertices(path.vertices().size());
for (size_t i = 0; i < path.vertices().size(); ++i) {
auto vertex_builder = vertices_builder[i];
storage::SaveVertexAccessor(path.vertices()[i], &vertex_builder,
versions, worker_id);
}
auto edges_builder = path_builder.initEdges(path.edges().size());
for (size_t i = 0; i < path.edges().size(); ++i) {
auto edge_builder = edges_builder[i];
storage::SaveEdgeAccessor(path.edges()[i], &edge_builder, versions,
worker_id);
}
return;
}
}
}
void LoadCapnpTypedValue(const capnp::TypedValue::Reader &reader,
TypedValue *value, database::GraphDbAccessor *dba,
distributed::DataManager *data_manager) {
switch (reader.which()) {
case capnp::TypedValue::NULL_TYPE:
*value = TypedValue::Null;
return;
case capnp::TypedValue::BOOL:
*value = reader.getBool();
return;
case capnp::TypedValue::INTEGER:
*value = reader.getInteger();
return;
case capnp::TypedValue::DOUBLE:
*value = reader.getDouble();
return;
case capnp::TypedValue::STRING:
*value = reader.getString().cStr();
return;
case capnp::TypedValue::LIST: {
std::vector<TypedValue> list;
list.reserve(reader.getList().size());
for (const auto &value_reader : reader.getList()) {
list.emplace_back();
LoadCapnpTypedValue(value_reader, &list.back(), dba, data_manager);
}
*value = list;
return;
}
case capnp::TypedValue::MAP: {
std::map<std::string, TypedValue> map;
for (const auto &kv_reader : reader.getMap()) {
auto key = kv_reader.getKey().cStr();
LoadCapnpTypedValue(kv_reader.getValue(), &map[key], dba, data_manager);
}
*value = map;
return;
}
case capnp::TypedValue::VERTEX:
*value =
storage::LoadVertexAccessor(reader.getVertex(), dba, data_manager);
return;
case capnp::TypedValue::EDGE:
*value = storage::LoadEdgeAccessor(reader.getEdge(), dba, data_manager);
return;
case capnp::TypedValue::PATH: {
auto vertices_reader = reader.getPath().getVertices();
auto edges_reader = reader.getPath().getEdges();
query::Path path(
storage::LoadVertexAccessor(vertices_reader[0], dba, data_manager));
for (size_t i = 0; i < edges_reader.size(); ++i) {
path.Expand(
storage::LoadEdgeAccessor(edges_reader[i], dba, data_manager));
path.Expand(storage::LoadVertexAccessor(vertices_reader[i + 1], dba,
data_manager));
}
*value = path;
return;
}
}
}
void Save(const Parameters &parameters,
utils::capnp::Map<utils::capnp::BoxInt64,
storage::capnp::PropertyValue>::Builder *builder) {
auto params_builder = builder->initEntries(parameters.size());
size_t i = 0;
for (auto &entry : parameters) {
auto builder = params_builder[i];
auto key_builder = builder.initKey();
key_builder.setValue(entry.first);
auto value_builder = builder.initValue();
storage::SaveCapnpPropertyValue(entry.second, &value_builder);
++i;
}
}
void Load(
Parameters *parameters,
const utils::capnp::Map<utils::capnp::BoxInt64,
storage::capnp::PropertyValue>::Reader &reader) {
for (const auto &entry_reader : reader.getEntries()) {
PropertyValue value;
storage::LoadCapnpPropertyValue(entry_reader.getValue(), &value);
parameters->Add(entry_reader.getKey().getValue(), value);
}
}
void Save(const TypedValueVectorCompare &comparator,
capnp::TypedValueVectorCompare::Builder *builder) {
auto ordering_builder = builder->initOrdering(comparator.ordering().size());
for (size_t i = 0; i < comparator.ordering().size(); ++i) {
ordering_builder.set(i, comparator.ordering()[i] == Ordering::ASC
? capnp::Ordering::ASC
: capnp::Ordering::DESC);
}
}
void Load(TypedValueVectorCompare *comparator,
const capnp::TypedValueVectorCompare::Reader &reader) {
std::vector<Ordering> ordering;
ordering.reserve(reader.getOrdering().size());
for (auto ordering_reader : reader.getOrdering()) {
ordering.push_back(ordering_reader == capnp::Ordering::ASC
? Ordering::ASC
: Ordering::DESC);
}
comparator->ordering_ = ordering;
}
} // namespace query
namespace slk { namespace slk {

56
src/query/distributed/serialization.hpp
View File

@ -4,69 +4,13 @@
#include "query/context.hpp" #include "query/context.hpp"
#include "query/distributed/frontend/semantic/symbol_serialization.hpp" #include "query/distributed/frontend/semantic/symbol_serialization.hpp"
#include "query/frontend/semantic/symbol_table.hpp" #include "query/frontend/semantic/symbol_table.hpp"
#include "query/distributed/serialization.capnp.h"
#include "query/typed_value.hpp" #include "query/typed_value.hpp"
#include "rpc/serialization.hpp"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
namespace distributed { namespace distributed {
class DataManager; class DataManager;
} }
namespace query {
void SaveCapnpTypedValue(const query::TypedValue &value,
capnp::TypedValue::Builder *builder,
storage::SendVersions versions, int worker_id);
void LoadCapnpTypedValue(const capnp::TypedValue::Reader &reader,
query::TypedValue *value,
database::GraphDbAccessor *dba,
distributed::DataManager *data_manager);
void Save(const TypedValueVectorCompare &comparator,
capnp::TypedValueVectorCompare::Builder *builder);
void Load(TypedValueVectorCompare *comparator,
const capnp::TypedValueVectorCompare::Reader &reader);
inline void Save(const SymbolTable &symbol_table,
capnp::SymbolTable::Builder *builder) {
auto table_builder = builder->initTable();
utils::SaveMap<utils::capnp::BoxInt32, capnp::Symbol,
std::map<int32_t, Symbol>>(
symbol_table.table(), &table_builder,
[](auto *builder, const auto &entry) {
auto key_builder = builder->initKey();
key_builder.setValue(entry.first);
auto value_builder = builder->initValue();
Save(entry.second, &value_builder);
});
}
inline void Load(SymbolTable *symbol_table,
const capnp::SymbolTable::Reader &reader) {
utils::LoadMap<utils::capnp::BoxInt32, capnp::Symbol,
std::map<int32_t, Symbol>>(
&symbol_table->table_, reader.getTable(), [](const auto &reader) {
std::pair<int32_t, Symbol> entry;
entry.first = reader.getKey().getValue();
Load(&entry.second, reader.getValue());
return entry;
});
}
void Save(const Parameters &parameters,
utils::capnp::Map<utils::capnp::BoxInt64,
storage::capnp::PropertyValue>::Builder *builder);
void Load(
Parameters *parameters,
const utils::capnp::Map<utils::capnp::BoxInt64,
storage::capnp::PropertyValue>::Reader &reader);
} // namespace query
namespace slk { namespace slk {
inline void Save(const query::SymbolTable &symbol_table, inline void Save(const query::SymbolTable &symbol_table,

631
src/query/frontend/ast/ast.lcp
View File

File diff suppressed because it is too large Load Diff

4
src/query/frontend/semantic/symbol.lcp
View File

@ -8,8 +8,6 @@ cpp<#
(lcp:namespace query) (lcp:namespace query)
(lcp:capnp-namespace "query")
(lcp:define-class symbol () (lcp:define-class symbol ()
((name "std::string" :scope :public) ((name "std::string" :scope :public)
(position :int64_t :scope :public) (position :int64_t :scope :public)
@ -53,7 +51,7 @@ cpp<#
bool user_declared() const { return user_declared_; } bool user_declared() const { return user_declared_; }
int token_position() const { return token_position_; } int token_position() const { return token_position_; }
cpp<#) cpp<#)
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:pop-namespace) ;; query (lcp:pop-namespace) ;; query

565
src/query/plan/operator.lcp
View File

File diff suppressed because it is too large Load Diff

9
src/raft/log_entry.lcp
View File

@ -3,22 +3,17 @@
#include "database/single_node_ha/serialization.hpp" #include "database/single_node_ha/serialization.hpp"
#include "durability/single_node_ha/state_delta.hpp" #include "durability/single_node_ha/state_delta.hpp"
#include "raft/log_entry.capnp.h"
cpp<# cpp<#
(lcp:namespace raft) (lcp:namespace raft)
(lcp:capnp-namespace "raft")
(lcp:capnp-import 'database "/database/single_node_ha/serialization.capnp")
(lcp:define-struct log-entry () (lcp:define-struct log-entry ()
((term :uint64_t) ((term :uint64_t)
(deltas "std::vector<database::StateDelta>" :capnp-type "List(Database.StateDelta)")) (deltas "std::vector<database::StateDelta>"))
(:public #>cpp (:public #>cpp
LogEntry() = default; LogEntry() = default;
LogEntry(uint64_t _term, std::vector<database::StateDelta> _deltas): term(_term), deltas(_deltas) {} LogEntry(uint64_t _term, std::vector<database::StateDelta> _deltas): term(_term), deltas(_deltas) {}
cpp<#) cpp<#)
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:pop-namespace) ;; raft (lcp:pop-namespace) ;; raft

10
src/raft/raft_rpc_messages.lcp
View File

@ -6,17 +6,11 @@
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "raft/log_entry.hpp" #include "raft/log_entry.hpp"
#include "raft/raft_rpc_messages.capnp.h"
#include "raft/snapshot_metadata.hpp" #include "raft/snapshot_metadata.hpp"
cpp<# cpp<#
(lcp:namespace raft) (lcp:namespace raft)
(lcp:capnp-namespace "raft")
(lcp:capnp-import 'log "/raft/log_entry.capnp")
(lcp:capnp-import 'snap "/raft/snapshot_metadata.capnp")
(lcp:define-rpc request-vote (lcp:define-rpc request-vote
(:request (:request
((candidate-id :uint16_t) ((candidate-id :uint16_t)
@ -34,7 +28,7 @@ cpp<#
(term :uint64_t) (term :uint64_t)
(prev-log-index :uint64_t) (prev-log-index :uint64_t)
(prev-log-term :uint64_t) (prev-log-term :uint64_t)
(entries "std::vector<raft::LogEntry>" :capnp-type "List(Log.LogEntry)"))) (entries "std::vector<raft::LogEntry>")))
(:response (:response
((success :bool) ((success :bool)
(term :uint64_t)))) (term :uint64_t))))
@ -51,7 +45,7 @@ cpp<#
(:request (:request
((leader-id :uint16_t) ((leader-id :uint16_t)
(term :uint64_t) (term :uint64_t)
(snapshot-metadata "raft::SnapshotMetadata" :capnp-type "Snap.SnapshotMetadata") (snapshot-metadata "raft::SnapshotMetadata")
(data "std::string"))) (data "std::string")))
(:response (:response
((term :uint64_t)))) ((term :uint64_t))))

1
src/raft/raft_server.cpp
View File

@ -15,7 +15,6 @@
#include "durability/single_node_ha/recovery.hpp" #include "durability/single_node_ha/recovery.hpp"
#include "durability/single_node_ha/snapshooter.hpp" #include "durability/single_node_ha/snapshooter.hpp"
#include "raft/exceptions.hpp" #include "raft/exceptions.hpp"
#include "rpc/serialization.hpp"
#include "slk/streams.hpp" #include "slk/streams.hpp"
#include "utils/cast.hpp" #include "utils/cast.hpp"
#include "utils/exceptions.hpp" #include "utils/exceptions.hpp"

5
src/raft/snapshot_metadata.lcp
View File

@ -2,14 +2,11 @@
#pragma once #pragma once
#include <string> #include <string>
#include "raft/snapshot_metadata.capnp.h"
#include "utils/typeinfo.hpp" #include "utils/typeinfo.hpp"
cpp<# cpp<#
(lcp:namespace raft) (lcp:namespace raft)
(lcp:capnp-namespace "raft")
(lcp:define-struct snapshot-metadata () (lcp:define-struct snapshot-metadata ()
((last-included-term :uint64_t) ((last-included-term :uint64_t)
(last-included-index :uint64_t) (last-included-index :uint64_t)
@ -22,6 +19,6 @@ cpp<#
last_included_index(_last_included_index), last_included_index(_last_included_index),
snapshot_filename(_snapshot_filename) {} snapshot_filename(_snapshot_filename) {}
cpp<#) cpp<#)
(:serialize (:slk) (:capnp))) (:serialize (:slk)))
(lcp:pop-namespace) ;; raft (lcp:pop-namespace) ;; raft

26
src/raft/storage_info_rpc_messages.lcp
View File

@ -5,39 +5,15 @@
#include <string> #include <string>
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "raft/storage_info_rpc_messages.capnp.h"
#include "rpc/serialization.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
cpp<# cpp<#
(lcp:namespace raft) (lcp:namespace raft)
(lcp:capnp-namespace "raft")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:define-rpc storage-info (lcp:define-rpc storage-info
(:request ()) (:request ())
(:response (:response
((server-id :uint16_t) ((server-id :uint16_t)
(storage-info "std::vector<std::pair<std::string, std::string>>" (storage-info "std::vector<std::pair<std::string, std::string>>"))))
:capnp-type "List(Utils.Pair(Text, Text))"
:capnp-save
(lcp:capnp-save-vector
"utils::capnp::Pair<::capnp::Text, ::capnp::Text>"
"std::pair<std::string, std::string>"
"[](auto *builder, const auto &pair) {
builder->setFirst(pair.first);
builder->setSecond(pair.second);
}")
:capnp-load
(lcp:capnp-load-vector
"utils::capnp::Pair<::capnp::Text, ::capnp::Text>"
"std::pair<std::string, std::string>"
"[](const auto &reader) {
std::string first = reader.getFirst();
std::string second = reader.getSecond();
return std::make_pair(first, second);
}")))))
(lcp:pop-namespace) ;; raft (lcp:pop-namespace) ;; raft

9
src/rpc/CMakeLists.txt
View File

@ -1,9 +0,0 @@
define_add_capnp(add_capnp rpc_src_files rpc_capnp_files)
add_capnp(serialization.capnp)
add_custom_target(generate_rpc_capnp DEPENDS ${rpc_capnp_files})
add_library(mg-rpc STATIC ${rpc_src_files})
target_link_libraries(mg-rpc glog gflags capnp kj)
add_dependencies(mg-rpc generate_rpc_capnp)

97
src/rpc/serialization.capnp
View File

@ -1,97 +0,0 @@
@0xe7647d63b36c2c65;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("utils::capnp");
# Primitive type wrappers
struct BoxInt16 {
value @0 :Int16;
}
struct BoxInt32 {
value @0 :Int32;
}
struct BoxInt64 {
value @0 :Int64;
}
struct BoxUInt16 {
value @0 :UInt16;
}
struct BoxUInt32 {
value @0 :UInt32;
}
struct BoxUInt64 {
value @0 :UInt64;
}
struct BoxFloat32 {
value @0 :Float32;
}
struct BoxFloat64 {
value @0 :Float64;
}
struct BoxBool {
value @0 :Bool;
}
# C++ STL types
struct Optional(T) {
union {
nullopt @0 :Void;
value @1 :T;
}
}
struct UniquePtr(T) {
union {
nullptr @0 :Void;
value @1 :T;
}
}
struct SharedPtr(T) {
union {
nullptr @0 :Void;
entry @1 :Entry;
}
struct Entry {
id @0 :UInt64;
value @1 :T;
}
}
struct Map(K, V) {
entries @0 :List(Entry);
struct Entry {
key @0 :K;
value @1 :V;
}
}
struct Pair(First, Second) {
first @0 :First;
second @1 :Second;
}
# Our types
struct Bound(T) {
type @0 :Type;
value @1 :T;
enum Type {
inclusive @0;
exclusive @1;
}
}

194
src/rpc/serialization.hpp
View File

@ -1,194 +0,0 @@
#pragma once
#include <limits>
#include <optional>
#include <vector>
#include <glog/logging.h>
#include "rpc/serialization.capnp.h"
#include "utils/algorithm.hpp"
namespace utils {
template <typename T>
inline void SaveVector(const std::vector<T> &data,
typename ::capnp::List<T>::Builder *list_builder) {
for (size_t i = 0; i < data.size(); ++i) {
list_builder->set(i, data[i]);
}
}
inline void SaveVector(const std::vector<std::string> &data,
::capnp::List<::capnp::Text>::Builder *list_builder) {
for (size_t i = 0; i < data.size(); ++i) {
list_builder->set(i, data[i]);
}
}
template <typename T>
inline void LoadVector(std::vector<T> *data,
const typename ::capnp::List<T>::Reader &list_reader) {
for (const auto e : list_reader) {
data->emplace_back(e);
}
}
inline void LoadVector(
std::vector<std::string> *data,
const typename ::capnp::List<::capnp::Text>::Reader &list_reader) {
for (const auto e : list_reader) {
data->emplace_back(e);
}
}
template <typename TCapnp, typename T>
inline void SaveVector(
const std::vector<T> &data,
typename ::capnp::List<TCapnp>::Builder *list_builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
for (size_t i = 0; i < data.size(); ++i) {
auto elem_builder = (*list_builder)[i];
save(&elem_builder, data[i]);
}
}
template <typename TCapnp, typename T>
inline void LoadVector(
std::vector<T> *data,
const typename ::capnp::List<TCapnp>::Reader &list_reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load) {
for (const auto reader : list_reader) {
data->emplace_back(load(reader));
}
}
template <class TCapnpKey, class TCapnpValue, class TMap>
void SaveMap(const TMap &map,
typename capnp::Map<TCapnpKey, TCapnpValue>::Builder *map_builder,
std::function<void(
typename capnp::Map<TCapnpKey, TCapnpValue>::Entry::Builder *,
const typename TMap::value_type &)>
save) {
auto entries_builder = map_builder->initEntries(map.size());
size_t i = 0;
for (const auto &entry : map) {
auto entry_builder = entries_builder[i];
save(&entry_builder, entry);
++i;
}
}
template <class TCapnpKey, class TCapnpValue, class TMap>
void LoadMap(
TMap *map,
const typename capnp::Map<TCapnpKey, TCapnpValue>::Reader &map_reader,
std::function<typename TMap::value_type(
const typename capnp::Map<TCapnpKey, TCapnpValue>::Entry::Reader &)>
load) {
for (const auto &entry_reader : map_reader.getEntries()) {
map->insert(load(entry_reader));
}
}
template <typename TCapnp, typename T>
inline void SaveOptional(
const std::optional<T> &data,
typename capnp::Optional<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
if (data) {
auto value_builder = builder->initValue();
save(&value_builder, data.value());
} else {
builder->setNullopt();
}
}
template <typename TCapnp, typename T>
inline std::optional<T> LoadOptional(
const typename capnp::Optional<TCapnp>::Reader &reader,
const std::function<T(const typename TCapnp::Reader &reader)> &load) {
switch (reader.which()) {
case capnp::Optional<TCapnp>::NULLOPT:
return std::nullopt;
case capnp::Optional<TCapnp>::VALUE:
auto value_reader = reader.getValue();
return std::optional<T>{load(value_reader)};
}
}
template <typename TCapnp, typename T>
inline void SaveUniquePtr(
const std::unique_ptr<T> &data,
typename capnp::UniquePtr<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save) {
if (data) {
auto value_builder = builder->initValue();
save(&value_builder, *data);
} else {
builder->setNullptr();
}
}
template <typename TCapnp, typename T>
inline std::unique_ptr<T> LoadUniquePtr(
const typename capnp::UniquePtr<TCapnp>::Reader &reader,
const std::function<T *(const typename TCapnp::Reader &reader)> &load) {
switch (reader.which()) {
case capnp::UniquePtr<TCapnp>::NULLPTR:
return nullptr;
case capnp::UniquePtr<TCapnp>::VALUE:
auto value_reader = reader.getValue();
return std::unique_ptr<T>(load(value_reader));
}
}
template <typename TCapnp, typename T>
inline void SaveSharedPtr(
const std::shared_ptr<T> &data,
typename capnp::SharedPtr<TCapnp>::Builder *builder,
const std::function<void(typename TCapnp::Builder *, const T &)> &save,
std::vector<T *> *saved_pointers) {
if (!data) {
builder->setNullptr();
return;
}
auto entry_builder = builder->initEntry();
auto pointer_id = reinterpret_cast<uintptr_t>(data.get());
CHECK(pointer_id <= std::numeric_limits<uint64_t>::max());
entry_builder.setId(pointer_id);
if (utils::Contains(*saved_pointers, data.get())) {
return;
}
auto value_builder = entry_builder.initValue();
save(&value_builder, *data);
saved_pointers->emplace_back(data.get());
}
template <typename TCapnp, typename T>
std::shared_ptr<T> LoadSharedPtr(
const typename capnp::SharedPtr<TCapnp>::Reader &reader,
const std::function<T *(const typename TCapnp::Reader &reader)> &load,
std::vector<std::pair<uint64_t, std::shared_ptr<T>>> *loaded_pointers) {
std::shared_ptr<T> ret;
switch (reader.which()) {
case capnp::SharedPtr<TCapnp>::NULLPTR:
ret = nullptr;
break;
case capnp::SharedPtr<TCapnp>::ENTRY:
auto entry_reader = reader.getEntry();
uint64_t pointer_id = entry_reader.getId();
auto found =
std::find_if(loaded_pointers->begin(), loaded_pointers->end(),
[pointer_id](const auto &e) -> bool {
return e.first == pointer_id;
});
if (found != loaded_pointers->end()) return found->second;
auto value_reader = entry_reader.getValue();
ret = std::shared_ptr<T>(load(value_reader));
loaded_pointers->emplace_back(std::make_pair(pointer_id, ret));
}
return ret;
}
} // namespace utils

7
src/stats/CMakeLists.txt
View File

@ -2,17 +2,12 @@ set(stats_src_files
metrics.cpp metrics.cpp
stats.cpp) stats.cpp)
define_add_capnp(add_capnp stats_src_files stats_capnp_files)
define_add_lcp(add_lcp stats_src_files stats_lcp_files) define_add_lcp(add_lcp stats_src_files stats_lcp_files)
add_lcp(stats_rpc_messages.lcp CAPNP_SCHEMA @0xc19a87c81b9b4512) add_lcp(stats_rpc_messages.lcp SLK_SERIALIZE)
add_capnp(stats_rpc_messages.capnp)
add_custom_target(generate_stats_lcp DEPENDS ${stats_lcp_files}) add_custom_target(generate_stats_lcp DEPENDS ${stats_lcp_files})
add_custom_target(generate_stats_capnp DEPENDS generate_stats_lcp ${stats_capnp_files})
add_library(mg-stats STATIC ${stats_src_files}) add_library(mg-stats STATIC ${stats_src_files})
target_link_libraries(mg-stats Threads::Threads mg-utils mg-io mg-comm-rpc fmt glog gflags) target_link_libraries(mg-stats Threads::Threads mg-utils mg-io mg-comm-rpc fmt glog gflags)
target_link_libraries(mg-stats capnp kj)
add_dependencies(mg-stats generate_stats_lcp) add_dependencies(mg-stats generate_stats_lcp)
add_dependencies(mg-stats generate_stats_capnp)

26
src/stats/stats_rpc_messages.lcp
View File

@ -3,39 +3,15 @@
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
#include "stats/stats_rpc_messages.capnp.h"
#include "rpc/serialization.hpp"
#include "utils/timestamp.hpp" #include "utils/timestamp.hpp"
cpp<# cpp<#
(lcp:namespace stats) (lcp:namespace stats)
(lcp:capnp-namespace "stats")
(lcp:capnp-import 'utils "/rpc/serialization.capnp")
(lcp:define-rpc stats (lcp:define-rpc stats
(:request (:request
((metric-path "std::string") ((metric-path "std::string")
(tags "std::vector<std::pair<std::string, std::string>>" (tags "std::vector<std::pair<std::string, std::string>>")
:capnp-type "List(Utils.Pair(Text, Text))"
:capnp-save
(lcp:capnp-save-vector
"utils::capnp::Pair<::capnp::Text, ::capnp::Text>"
"std::pair<std::string, std::string>"
"[](auto *builder, const auto &pair) {
builder->setFirst(pair.first);
builder->setSecond(pair.second);
}")
:capnp-load
(lcp:capnp-load-vector
"utils::capnp::Pair<::capnp::Text, ::capnp::Text>"
"std::pair<std::string, std::string>"
"[](const auto &reader) {
std::string first = reader.getFirst();
std::string second = reader.getSecond();
return std::make_pair(first, second);
}"))
(value :double) (value :double)
(timestamp :uint64_t :initarg nil (timestamp :uint64_t :initarg nil
:initval "static_cast<uint64_t>(utils::Timestamp::Now().SecSinceTheEpoch())"))) :initval "static_cast<uint64_t>(utils::Timestamp::Now().SecSinceTheEpoch())")))

17
src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.lcp
View File

@ -5,7 +5,6 @@
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "storage/common/types/types.hpp" #include "storage/common/types/types.hpp"
#include "storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp.h"
#include "storage/distributed/rpc/serialization.hpp" #include "storage/distributed/rpc/serialization.hpp"
#include "transactions/commit_log.hpp" #include "transactions/commit_log.hpp"
#include "transactions/snapshot.hpp" #include "transactions/snapshot.hpp"
@ -14,32 +13,28 @@ cpp<#
(lcp:namespace storage) (lcp:namespace storage)
(lcp:capnp-namespace "storage")
(lcp:capnp-import 's "/storage/distributed/rpc/serialization.capnp")
(lcp:define-rpc label-id (lcp:define-rpc label-id
(:request ((member "std::string"))) (:request ((member "std::string")))
(:response ((member "Label" :capnp-type "S.Label")))) (:response ((member "Label"))))
(lcp:define-rpc id-label (lcp:define-rpc id-label
(:request ((member "Label" :capnp-type "S.Label"))) (:request ((member "Label")))
(:response ((member "std::string")))) (:response ((member "std::string"))))
(lcp:define-rpc edge-type-id (lcp:define-rpc edge-type-id
(:request ((member "std::string"))) (:request ((member "std::string")))
(:response ((member "EdgeType" :capnp-type "S.EdgeType")))) (:response ((member "EdgeType"))))
(lcp:define-rpc id-edge-type (lcp:define-rpc id-edge-type
(:request ((member "EdgeType" :capnp-type "S.EdgeType"))) (:request ((member "EdgeType")))
(:response ((member "std::string")))) (:response ((member "std::string"))))
(lcp:define-rpc property-id (lcp:define-rpc property-id
(:request ((member "std::string"))) (:request ((member "std::string")))
(:response ((member "Property" :capnp-type "S.Property")))) (:response ((member "Property"))))
(lcp:define-rpc id-property (lcp:define-rpc id-property
(:request ((member "Property" :capnp-type "S.Property"))) (:request ((member "Property")))
(:response ((member "std::string")))) (:response ((member "std::string"))))
(lcp:pop-namespace) (lcp:pop-namespace)

87
src/storage/distributed/rpc/serialization.capnp
View File

@ -1,87 +0,0 @@
@0x8678c6a8817808d9;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("storage::capnp");
struct Label {
storage @0 :UInt16;
}
struct EdgeType {
storage @0 :UInt16;
}
struct Property {
storage @0 :UInt16;
}
struct Address {
storage @0 :UInt64;
}
struct PropertyValue {
union {
nullType @0 :Void;
bool @1 :Bool;
integer @2 :Int64;
double @3 :Float64;
string @4 :Text;
list @5 :List(PropertyValue);
map @6 :List(MapEntry);
}
struct MapEntry {
key @0 :Text;
value @1 :PropertyValue;
}
}
struct PropertyValueStore {
properties @0 :List(Entry);
struct Entry {
id @0 :Property;
value @1 :PropertyValue;
}
}
struct Edge {
from @0 :Address;
to @1 :Address;
typeId @2 :UInt16;
properties @3 :PropertyValueStore;
}
struct Vertex {
outEdges @0 :List(EdgeEntry);
inEdges @1 :List(EdgeEntry);
labelIds @2 :List(UInt16);
properties @3 :PropertyValueStore;
struct EdgeEntry {
vertexAddress @0 :Address;
edgeAddress @1 :Address;
edgeTypeId @2 :UInt16;
}
}
enum SendVersions {
both @0;
onlyOld @1;
onlyNew @2;
}
struct VertexAccessor {
cypherId @0 :Int64;
address @1 :UInt64;
old @2 :Vertex;
new @3 :Vertex;
}
struct EdgeAccessor {
cypherId @0 :Int64;
address @1 :UInt64;
old @2 :Edge;
new @3 :Edge;
}

312
src/storage/distributed/rpc/serialization.cpp
View File

@ -3,318 +3,6 @@
#include "database/distributed/graph_db_accessor.hpp" #include "database/distributed/graph_db_accessor.hpp"
#include "distributed/data_manager.hpp" #include "distributed/data_manager.hpp"
namespace storage {
void Save(const Label &label, capnp::Label::Builder *builder) {
builder->setStorage(label.id_);
}
void Load(Label *label, const capnp::Label::Reader &reader) {
label->id_ = reader.getStorage();
}
void Save(const EdgeType &edge_type, capnp::EdgeType::Builder *builder) {
builder->setStorage(edge_type.id_);
}
void Load(EdgeType *edge_type, const capnp::EdgeType::Reader &reader) {
edge_type->id_ = reader.getStorage();
}
void Save(const Property &property, capnp::Property::Builder *builder) {
builder->setStorage(property.id_);
}
void Load(Property *property, const capnp::Property::Reader &reader) {
property->id_ = reader.getStorage();
}
void SaveCapnpPropertyValue(const PropertyValue &value,
capnp::PropertyValue::Builder *builder) {
switch (value.type()) {
case PropertyValue::Type::Null:
builder->setNullType();
return;
case PropertyValue::Type::Bool:
builder->setBool(value.Value<bool>());
return;
case PropertyValue::Type::Int:
builder->setInteger(value.Value<int64_t>());
return;
case PropertyValue::Type::Double:
builder->setDouble(value.Value<double>());
return;
case PropertyValue::Type::String:
builder->setString(value.Value<std::string>());
return;
case PropertyValue::Type::List: {
const auto &values = value.Value<std::vector<PropertyValue>>();
auto list_builder = builder->initList(values.size());
for (size_t i = 0; i < values.size(); ++i) {
auto value_builder = list_builder[i];
SaveCapnpPropertyValue(values[i], &value_builder);
}
return;
}
case PropertyValue::Type::Map: {
const auto &map = value.Value<std::map<std::string, PropertyValue>>();
auto map_builder = builder->initMap(map.size());
size_t i = 0;
for (const auto &kv : map) {
auto kv_builder = map_builder[i];
kv_builder.setKey(kv.first);
auto value_builder = kv_builder.initValue();
SaveCapnpPropertyValue(kv.second, &value_builder);
++i;
}
return;
}
}
}
void LoadCapnpPropertyValue(const capnp::PropertyValue::Reader &reader,
PropertyValue *value) {
switch (reader.which()) {
case capnp::PropertyValue::NULL_TYPE:
*value = PropertyValue::Null;
return;
case capnp::PropertyValue::BOOL:
*value = reader.getBool();
return;
case capnp::PropertyValue::INTEGER:
*value = reader.getInteger();
return;
case capnp::PropertyValue::DOUBLE:
*value = reader.getDouble();
return;
case capnp::PropertyValue::STRING:
*value = reader.getString().cStr();
return;
case capnp::PropertyValue::LIST: {
std::vector<PropertyValue> list;
list.reserve(reader.getList().size());
for (const auto &value_reader : reader.getList()) {
list.emplace_back();
LoadCapnpPropertyValue(value_reader, &list.back());
}
*value = list;
return;
}
case capnp::PropertyValue::MAP: {
std::map<std::string, PropertyValue> map;
for (const auto &kv_reader : reader.getMap()) {
auto key = kv_reader.getKey();
LoadCapnpPropertyValue(kv_reader.getValue(), &map[key]);
}
*value = map;
return;
}
}
}
void SaveProperties(const PropertyValueStore &properties,
capnp::PropertyValueStore::Builder *builder) {
size_t i = 0;
auto props_builder = builder->initProperties(properties.size());
for (const auto &kv : properties) {
auto kv_builder = props_builder[i++];
auto id_builder = kv_builder.initId();
Save(kv.first, &id_builder);
auto value_builder = kv_builder.initValue();
SaveCapnpPropertyValue(kv.second, &value_builder);
}
}
void LoadProperties(const capnp::PropertyValueStore::Reader &reader,
PropertyValueStore *properties) {
properties->clear();
auto props_reader = reader.getProperties();
for (const auto &kv_reader : props_reader) {
storage::Property id;
storage::Load(&id, kv_reader.getId());
PropertyValue value;
LoadCapnpPropertyValue(kv_reader.getValue(), &value);
properties->set(id, value);
}
}
template <class TAddress>
void SaveAddress(TAddress address, capnp::Address::Builder *builder,
int16_t worker_id) {
TAddress global_address =
address.is_local() ? TAddress(address.local()->gid_, worker_id) : address;
builder->setStorage(global_address.raw());
}
EdgeAddress LoadEdgeAddress(const capnp::Address::Reader &reader) {
return EdgeAddress(reader.getStorage());
}
VertexAddress LoadVertexAddress(const capnp::Address::Reader &reader) {
return VertexAddress(reader.getStorage());
}
void SaveVertex(const Vertex &vertex, capnp::Vertex::Builder *builder,
int16_t worker_id) {
auto save_edges = [worker_id](const auto &edges, auto *edges_builder) {
int64_t i = 0;
for (const auto &edge : edges) {
auto edge_builder = (*edges_builder)[i];
auto vertex_addr_builder = edge_builder.initVertexAddress();
SaveAddress(edge.vertex, &vertex_addr_builder, worker_id);
auto edge_addr_builder = edge_builder.initEdgeAddress();
SaveAddress(edge.edge, &edge_addr_builder, worker_id);
edge_builder.setEdgeTypeId(edge.edge_type.Id());
++i;
}
};
auto out_builder = builder->initOutEdges(vertex.out_.size());
save_edges(vertex.out_, &out_builder);
auto in_builder = builder->initInEdges(vertex.in_.size());
save_edges(vertex.in_, &in_builder);
auto labels_builder = builder->initLabelIds(vertex.labels_.size());
for (size_t i = 0; i < vertex.labels_.size(); ++i) {
labels_builder.set(i, vertex.labels_[i].Id());
}
auto properties_builder = builder->initProperties();
storage::SaveProperties(vertex.properties_, &properties_builder);
}
void SaveEdge(const Edge &edge, capnp::Edge::Builder *builder,
int16_t worker_id) {
auto from_builder = builder->initFrom();
SaveAddress(edge.from_, &from_builder, worker_id);
auto to_builder = builder->initTo();
SaveAddress(edge.to_, &to_builder, worker_id);
builder->setTypeId(edge.edge_type_.Id());
auto properties_builder = builder->initProperties();
storage::SaveProperties(edge.properties_, &properties_builder);
}
/// Alias for `SaveEdge` allowing for param type resolution.
void SaveElement(const Edge &record, capnp::Edge::Builder *builder,
int16_t worker_id) {
return SaveEdge(record, builder, worker_id);
}
/// Alias for `SaveVertex` allowing for param type resolution.
void SaveElement(const Vertex &record, capnp::Vertex::Builder *builder,
int16_t worker_id) {
return SaveVertex(record, builder, worker_id);
}
std::unique_ptr<Vertex> LoadVertex(const capnp::Vertex::Reader &reader) {
auto vertex = std::make_unique<Vertex>();
auto load_edges = [](const auto &edges_reader) {
Edges edges;
for (const auto &edge_reader : edges_reader) {
auto vertex_address = LoadVertexAddress(edge_reader.getVertexAddress());
auto edge_address = LoadEdgeAddress(edge_reader.getEdgeAddress());
storage::EdgeType edge_type(edge_reader.getEdgeTypeId());
edges.emplace(vertex_address, edge_address, edge_type);
}
return edges;
};
vertex->out_ = load_edges(reader.getOutEdges());
vertex->in_ = load_edges(reader.getInEdges());
for (const auto &label_id : reader.getLabelIds()) {
vertex->labels_.emplace_back(label_id);
}
storage::LoadProperties(reader.getProperties(), &vertex->properties_);
return vertex;
}
std::unique_ptr<Edge> LoadEdge(const capnp::Edge::Reader &reader) {
auto from = LoadVertexAddress(reader.getFrom());
auto to = LoadVertexAddress(reader.getTo());
auto edge =
std::make_unique<Edge>(from, to, storage::EdgeType{reader.getTypeId()});
storage::LoadProperties(reader.getProperties(), &edge->properties_);
return edge;
}
template <class TRecord, class TCapnpRecord>
void SaveRecordAccessor(const RecordAccessor<TRecord> &accessor,
typename TCapnpRecord::Builder *builder,
SendVersions versions, int worker_id) {
builder->setCypherId(accessor.CypherId());
builder->setAddress(accessor.GlobalAddress().raw());
bool reconstructed = false;
auto guard = storage::GetDataLock(accessor);
if (!accessor.GetOld() && !accessor.GetNew()) {
reconstructed = true;
bool result = accessor.Reconstruct();
CHECK(result) << "Attempting to serialize an element not visible to "
"current transaction.";
}
auto old_rec = accessor.GetOld();
if (old_rec && versions != SendVersions::ONLY_NEW) {
auto old_builder = builder->initOld();
storage::SaveElement(*old_rec, &old_builder, worker_id);
}
if (versions != SendVersions::ONLY_OLD) {
if (!reconstructed && !accessor.GetNew()) {
bool result = accessor.Reconstruct();
CHECK(result) << "Attempting to serialize an element not visible to "
"current transaction.";
}
auto *new_rec = accessor.GetNew();
if (new_rec) {
auto new_builder = builder->initNew();
storage::SaveElement(*new_rec, &new_builder, worker_id);
}
}
}
void SaveVertexAccessor(const VertexAccessor &vertex_accessor,
capnp::VertexAccessor::Builder *builder,
SendVersions versions, int worker_id) {
SaveRecordAccessor<Vertex, capnp::VertexAccessor>(vertex_accessor, builder,
versions, worker_id);
}
void SaveEdgeAccessor(const EdgeAccessor &edge_accessor,
capnp::EdgeAccessor::Builder *builder,
SendVersions versions, int worker_id) {
SaveRecordAccessor<Edge, capnp::EdgeAccessor>(edge_accessor, builder,
versions, worker_id);
}
VertexAccessor LoadVertexAccessor(const capnp::VertexAccessor::Reader &reader,
database::GraphDbAccessor *dba,
distributed::DataManager *data_manager) {
int64_t cypher_id = reader.getCypherId();
storage::VertexAddress global_address(reader.getAddress());
auto old_record =
reader.hasOld() ? storage::LoadVertex(reader.getOld()) : nullptr;
auto new_record =
reader.hasNew() ? storage::LoadVertex(reader.getNew()) : nullptr;
data_manager->Emplace(
dba->transaction_id(), global_address.gid(),
distributed::CachedRecordData<Vertex>(cypher_id, std::move(old_record),
std::move(new_record)));
return VertexAccessor(global_address, *dba);
}
EdgeAccessor LoadEdgeAccessor(const capnp::EdgeAccessor::Reader &reader,
database::GraphDbAccessor *dba,
distributed::DataManager *data_manager) {
int64_t cypher_id = reader.getCypherId();
storage::EdgeAddress global_address(reader.getAddress());
auto old_record =
reader.hasOld() ? storage::LoadEdge(reader.getOld()) : nullptr;
auto new_record =
reader.hasNew() ? storage::LoadEdge(reader.getNew()) : nullptr;
data_manager->Emplace(
dba->transaction_id(), global_address.gid(),
distributed::CachedRecordData<Edge>(cypher_id, std::move(old_record),
std::move(new_record)));
return EdgeAccessor(global_address, *dba);
}
} // namespace storage
namespace slk { namespace slk {
void Save(const storage::SendVersions &versions, slk::Builder *builder) { void Save(const storage::SendVersions &versions, slk::Builder *builder) {

69
src/storage/distributed/rpc/serialization.hpp
View File

@ -6,7 +6,6 @@
#include "storage/common/types/slk.hpp" #include "storage/common/types/slk.hpp"
#include "storage/distributed/edge.hpp" #include "storage/distributed/edge.hpp"
#include "storage/distributed/edge_accessor.hpp" #include "storage/distributed/edge_accessor.hpp"
#include "storage/distributed/rpc/serialization.capnp.h"
#include "storage/distributed/vertex.hpp" #include "storage/distributed/vertex.hpp"
#include "storage/distributed/vertex_accessor.hpp" #include "storage/distributed/vertex_accessor.hpp"
@ -20,76 +19,8 @@ class DataManager;
namespace storage { namespace storage {
template <typename TLocalObj>
void Save(const Address<TLocalObj> &address, capnp::Address::Builder *builder) {
builder->setStorage(address.raw());
}
template <typename TLocalObj>
void Load(Address<TLocalObj> *address, const capnp::Address::Reader &reader) {
address->storage_ = reader.getStorage();
}
void Save(const Label &label, capnp::Label::Builder *builder);
void Load(Label *label, const capnp::Label::Reader &reader);
void Save(const EdgeType &edge_type, capnp::EdgeType::Builder *builder);
void Load(EdgeType *edge_type, const capnp::EdgeType::Reader &reader);
void Save(const Property &property, capnp::Property::Builder *builder);
void Load(Property *property, const capnp::Property::Reader &reader);
void SaveCapnpPropertyValue(const PropertyValue &value,
capnp::PropertyValue::Builder *builder);
void LoadCapnpPropertyValue(const capnp::PropertyValue::Reader &reader,
PropertyValue *value);
void SaveProperties(const PropertyValueStore &properties,
capnp::PropertyValueStore::Builder *builder);
void LoadProperties(const capnp::PropertyValueStore::Reader &reader,
PropertyValueStore *properties);
void SaveVertex(const Vertex &vertex, capnp::Vertex::Builder *builder,
int16_t worker_id);
void SaveEdge(const Edge &edge, capnp::Edge::Builder *builder,
int16_t worker_id);
/// Alias for `SaveEdge` allowing for param type resolution.
void SaveElement(const Edge &record, capnp::Edge::Builder *builder,
int16_t worker_id);
/// Alias for `SaveVertex` allowing for param type resolution.
void SaveElement(const Vertex &record, capnp::Vertex::Builder *builder,
int16_t worker_id);
std::unique_ptr<Vertex> LoadVertex(const capnp::Vertex::Reader &reader);
std::unique_ptr<Edge> LoadEdge(const capnp::Edge::Reader &reader);
enum class SendVersions { BOTH, ONLY_OLD, ONLY_NEW }; enum class SendVersions { BOTH, ONLY_OLD, ONLY_NEW };
void SaveVertexAccessor(const VertexAccessor &vertex_accessor,
capnp::VertexAccessor::Builder *builder,
SendVersions versions, int worker_id);
VertexAccessor LoadVertexAccessor(const capnp::VertexAccessor::Reader &reader,
database::GraphDbAccessor *dba,
distributed::DataManager *data_manager);
void SaveEdgeAccessor(const EdgeAccessor &edge_accessor,
capnp::EdgeAccessor::Builder *builder,
SendVersions versions, int worker_id);
EdgeAccessor LoadEdgeAccessor(const capnp::EdgeAccessor::Reader &reader,
database::GraphDbAccessor *dba,
distributed::DataManager *data_manager);
} // namespace storage } // namespace storage
namespace slk { namespace slk {

33
src/storage/single_node_ha/rpc/serialization.capnp
View File

@ -1,33 +0,0 @@
@0x8424471a44ccd2df;
using Cxx = import "/capnp/c++.capnp";
$Cxx.namespace("storage::capnp");
struct Label {
storage @0 :UInt16;
}
struct EdgeType {
storage @0 :UInt16;
}
struct Property {
storage @0 :UInt16;
}
struct PropertyValue {
union {
nullType @0 :Void;
bool @1 :Bool;
integer @2 :Int64;
double @3 :Float64;
string @4 :Text;
list @5 :List(PropertyValue);
map @6 :List(MapEntry);
}
struct MapEntry {
key @0 :Text;
value @1 :PropertyValue;
}
}

112
src/storage/single_node_ha/rpc/serialization.cpp
View File

@ -1,112 +0,0 @@
#include "storage/single_node_ha/rpc/serialization.hpp"
namespace storage {
void Save(const Label &label, capnp::Label::Builder *builder) {
builder->setStorage(label.id_);
}
void Load(Label *label, const capnp::Label::Reader &reader) {
label->id_ = reader.getStorage();
}
void Save(const EdgeType &edge_type, capnp::EdgeType::Builder *builder) {
builder->setStorage(edge_type.id_);
}
void Load(EdgeType *edge_type, const capnp::EdgeType::Reader &reader) {
edge_type->id_ = reader.getStorage();
}
void Save(const Property &property, capnp::Property::Builder *builder) {
builder->setStorage(property.id_);
}
void Load(Property *property, const capnp::Property::Reader &reader) {
property->id_ = reader.getStorage();
}
void SaveCapnpPropertyValue(const PropertyValue &value,
capnp::PropertyValue::Builder *builder) {
switch (value.type()) {
case PropertyValue::Type::Null:
builder->setNullType();
return;
case PropertyValue::Type::Bool:
builder->setBool(value.Value<bool>());
return;
case PropertyValue::Type::Int:
builder->setInteger(value.Value<int64_t>());
return;
case PropertyValue::Type::Double:
builder->setDouble(value.Value<double>());
return;
case PropertyValue::Type::String:
builder->setString(value.Value<std::string>());
return;
case PropertyValue::Type::List: {
const auto &values = value.Value<std::vector<PropertyValue>>();
auto list_builder = builder->initList(values.size());
for (size_t i = 0; i < values.size(); ++i) {
auto value_builder = list_builder[i];
SaveCapnpPropertyValue(values[i], &value_builder);
}
return;
}
case PropertyValue::Type::Map: {
const auto &map = value.Value<std::map<std::string, PropertyValue>>();
auto map_builder = builder->initMap(map.size());
size_t i = 0;
for (const auto &kv : map) {
auto kv_builder = map_builder[i];
kv_builder.setKey(kv.first);
auto value_builder = kv_builder.initValue();
SaveCapnpPropertyValue(kv.second, &value_builder);
++i;
}
return;
}
}
}
void LoadCapnpPropertyValue(const capnp::PropertyValue::Reader &reader,
PropertyValue *value) {
switch (reader.which()) {
case capnp::PropertyValue::NULL_TYPE:
*value = PropertyValue::Null;
return;
case capnp::PropertyValue::BOOL:
*value = reader.getBool();
return;
case capnp::PropertyValue::INTEGER:
*value = reader.getInteger();
return;
case capnp::PropertyValue::DOUBLE:
*value = reader.getDouble();
return;
case capnp::PropertyValue::STRING:
*value = reader.getString().cStr();
return;
case capnp::PropertyValue::LIST: {
std::vector<PropertyValue> list;
list.reserve(reader.getList().size());
for (const auto &value_reader : reader.getList()) {
list.emplace_back();
LoadCapnpPropertyValue(value_reader, &list.back());
}
*value = list;
return;
}
case capnp::PropertyValue::MAP: {
std::map<std::string, PropertyValue> map;
for (const auto &kv_reader : reader.getMap()) {
auto key = kv_reader.getKey();
LoadCapnpPropertyValue(kv_reader.getValue(), &map[key]);
}
*value = map;
return;
}
}
}
} // namespace storage

29
src/storage/single_node_ha/rpc/serialization.hpp
View File

@ -1,29 +0,0 @@
#pragma once
#include "slk/serialization.hpp"
#include "storage/common/types/property_value.hpp"
#include "storage/common/types/slk.hpp"
#include "storage/common/types/types.hpp"
#include "storage/single_node_ha/rpc/serialization.capnp.h"
namespace storage {
void Save(const Label &label, capnp::Label::Builder *builder);
void Load(Label *label, const capnp::Label::Reader &reader);
void Save(const EdgeType &edge_type, capnp::EdgeType::Builder *builder);
void Load(EdgeType *edge_type, const capnp::EdgeType::Reader &reader);
void Save(const Property &property, capnp::Property::Builder *builder);
void Load(Property *property, const capnp::Property::Reader &reader);
void SaveCapnpPropertyValue(const PropertyValue &value,
capnp::PropertyValue::Builder *builder);
void LoadCapnpPropertyValue(const capnp::PropertyValue::Reader &reader,
PropertyValue *value);
} // namespace storage

38
src/transactions/distributed/engine_rpc_messages.lcp
View File

@ -4,7 +4,6 @@
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
#include "transactions/commit_log.hpp" #include "transactions/commit_log.hpp"
#include "transactions/distributed/engine_rpc_messages.capnp.h"
#include "transactions/snapshot.hpp" #include "transactions/snapshot.hpp"
#include "transactions/type.hpp" #include "transactions/type.hpp"
cpp<# cpp<#
@ -13,21 +12,12 @@ cpp<#
(lcp:namespace tx) (lcp:namespace tx)
(lcp:capnp-namespace "tx")
(lcp:capnp-type-conversion "TransactionId" "UInt64")
(lcp:capnp-type-conversion "CommandId" "UInt32")
(lcp:define-struct tx-and-snapshot () (lcp:define-struct tx-and-snapshot ()
((tx-id "TransactionId") ((tx-id "TransactionId")
(snapshot "Snapshot" (snapshot "Snapshot"
:slk-save #'slk-save-snapshot :slk-save #'slk-save-snapshot
:slk-load #'slk-load-snapshot :slk-load #'slk-load-snapshot))
:capnp-type "List(UInt64)" (:serialize (:slk)))
:capnp-init nil
:capnp-save #'save-snapshot
:capnp-load #'load-snapshot))
(:serialize (:slk) (:capnp)))
(lcp:define-rpc begin (lcp:define-rpc begin
(:request ()) (:request ())
@ -49,11 +39,7 @@ cpp<#
(:request ((member "TransactionId"))) (:request ((member "TransactionId")))
(:response ((member "Snapshot" (:response ((member "Snapshot"
:slk-save #'slk-save-snapshot :slk-save #'slk-save-snapshot
:slk-load #'slk-load-snapshot :slk-load #'slk-load-snapshot))))
:capnp-type "List(UInt64)"
:capnp-init nil
:capnp-save #'save-snapshot
:capnp-load #'load-snapshot))))
(lcp:define-rpc command (lcp:define-rpc command
(:request ((member "TransactionId"))) (:request ((member "TransactionId")))
@ -63,31 +49,19 @@ cpp<#
(:request ()) (:request ())
(:response ((member "Snapshot" (:response ((member "Snapshot"
:slk-save #'slk-save-snapshot :slk-save #'slk-save-snapshot
:slk-load #'slk-load-snapshot :slk-load #'slk-load-snapshot))))
:capnp-type "List(UInt64)"
:capnp-init nil
:capnp-save #'save-snapshot
:capnp-load #'load-snapshot))))
(lcp:define-rpc clog-info (lcp:define-rpc clog-info
(:request ((member "TransactionId"))) (:request ((member "TransactionId")))
(:response ((member "CommitLog::Info" (:response ((member "CommitLog::Info"
:slk-save #'slk-save-commitlog-info :slk-save #'slk-save-commitlog-info
:slk-load #'slk-load-commitlog-info :slk-load #'slk-load-commitlog-info))))
:capnp-type "UInt8"
:capnp-init nil
:capnp-save #'save-commitlog-info
:capnp-load #'load-commitlog-info))))
(lcp:define-rpc active-transactions (lcp:define-rpc active-transactions
(:request ()) (:request ())
(:response ((member "Snapshot" (:response ((member "Snapshot"
:slk-save #'slk-save-snapshot :slk-save #'slk-save-snapshot
:slk-load #'slk-load-snapshot :slk-load #'slk-load-snapshot))))
:capnp-type "List(UInt64)"
:capnp-init nil
:capnp-save #'save-snapshot
:capnp-load #'load-snapshot))))
(lcp:define-rpc ensure-next-id-greater (lcp:define-rpc ensure-next-id-greater
(:request ((member "TransactionId"))) (:request ((member "TransactionId")))

23
src/transactions/distributed/serialization.lcp
View File

@ -12,16 +12,6 @@
self->${member} = tx::CommitLog::Info(info_flags); self->${member} = tx::CommitLog::Info(info_flags);
cpp<#) cpp<#)
(defun save-commitlog-info (builder member capnp-name)
#>cpp
${builder}->set${capnp-name}(${member});
cpp<#)
(defun load-commitlog-info (reader member capnp-name)
#>cpp
${member} = CommitLog::Info(${reader}.get${capnp-name}());
cpp<#)
(defun slk-save-snapshot (member) (defun slk-save-snapshot (member)
#>cpp #>cpp
slk::Save(self.${member}.transaction_ids(), builder); slk::Save(self.${member}.transaction_ids(), builder);
@ -33,16 +23,3 @@
slk::Load(&transaction_ids, reader); slk::Load(&transaction_ids, reader);
self->${member} = tx::Snapshot(std::move(transaction_ids)); self->${member} = tx::Snapshot(std::move(transaction_ids));
cpp<#) cpp<#)
(defun save-snapshot (builder member capnp-name)
#>cpp
auto list_builder = builder->init${capnp-name}(${member}.transaction_ids().size());
utils::SaveVector(${member}.transaction_ids(), &list_builder);
cpp<#)
(defun load-snapshot (reader member capnp-name)
#>cpp
std::vector<uint64_t> transaction_ids;
utils::LoadVector(&transaction_ids, ${reader}.get${capnp-name}());
${member} = tx::Snapshot(std::move(transaction_ids));
cpp<#)

2
tests/benchmark/rpc.cpp
View File

@ -3,8 +3,6 @@
#include <benchmark/benchmark.h> #include <benchmark/benchmark.h>
#include "capnp/serialize.h"
#include "communication/rpc/client.hpp" #include "communication/rpc/client.hpp"
#include "communication/rpc/client_pool.hpp" #include "communication/rpc/client_pool.hpp"
#include "communication/rpc/messages.hpp" #include "communication/rpc/messages.hpp"

69
tests/benchmark/serialization.cpp
View File

@ -3,9 +3,6 @@
#include <benchmark/benchmark.h> #include <benchmark/benchmark.h>
#include <capnp/serialize.h>
#include <kj/std/iostream.h>
#include "query/distributed/frontend/semantic/symbol_serialization.hpp" #include "query/distributed/frontend/semantic/symbol_serialization.hpp"
#include "query/frontend/semantic/symbol.hpp" #include "query/frontend/semantic/symbol.hpp"
#include "slk/serialization.hpp" #include "slk/serialization.hpp"
@ -30,72 +27,6 @@ class SymbolVectorFixture : public benchmark::Fixture {
void TearDown(const benchmark::State &) override { symbols_.clear(); } void TearDown(const benchmark::State &) override { symbols_.clear(); }
}; };
void SymbolVectorToCapnpMessage(const std::vector<query::Symbol> &symbols,
capnp::MessageBuilder &message) {
auto symbols_builder =
message.initRoot<capnp::List<query::capnp::Symbol>>(symbols.size());
for (int i = 0; i < symbols.size(); ++i) {
const auto &sym = symbols[i];
query::capnp::Symbol::Builder sym_builder = symbols_builder[i];
sym_builder.setName(sym.name());
sym_builder.setPosition(sym.position());
sym_builder.setType(query::capnp::Symbol::Type::ANY);
sym_builder.setUserDeclared(sym.user_declared());
sym_builder.setTokenPosition(sym.token_position());
}
}
std::stringstream SerializeCapnpSymbolVector(
const std::vector<query::Symbol> &symbols) {
std::stringstream stream(std::ios_base::in | std::ios_base::out |
std::ios_base::binary);
{
capnp::MallocMessageBuilder message;
SymbolVectorToCapnpMessage(symbols, message);
kj::std::StdOutputStream std_stream(stream);
kj::BufferedOutputStreamWrapper buffered_stream(std_stream);
writeMessage(buffered_stream, message);
}
return stream;
}
BENCHMARK_DEFINE_F(SymbolVectorFixture, CapnpSerial)(benchmark::State &state) {
while (state.KeepRunning()) {
SerializeCapnpSymbolVector(symbols_);
}
state.SetItemsProcessed(state.iterations());
}
BENCHMARK_DEFINE_F(SymbolVectorFixture, CapnpDeserial)
(benchmark::State &state) {
while (state.KeepRunning()) {
state.PauseTiming();
auto stream = SerializeCapnpSymbolVector(symbols_);
state.ResumeTiming();
kj::std::StdInputStream std_stream(stream);
capnp::InputStreamMessageReader message(std_stream);
auto symbols_reader = message.getRoot<capnp::List<query::capnp::Symbol>>();
std::vector<query::Symbol> symbols;
symbols.reserve(symbols_reader.size());
for (const auto &sym : symbols_reader) {
symbols.emplace_back(sym.getName().cStr(), sym.getPosition(),
sym.getUserDeclared(), query::Symbol::Type::ANY,
sym.getTokenPosition());
}
}
state.SetItemsProcessed(state.iterations());
}
BENCHMARK_REGISTER_F(SymbolVectorFixture, CapnpSerial)
->RangeMultiplier(4)
->Range(4, 1 << 12)
->Unit(benchmark::kNanosecond);
BENCHMARK_REGISTER_F(SymbolVectorFixture, CapnpDeserial)
->RangeMultiplier(4)
->Range(4, 1 << 12)
->Unit(benchmark::kNanosecond);
void SymbolVectorToSlk(const std::vector<query::Symbol> &symbols, void SymbolVectorToSlk(const std::vector<query::Symbol> &symbols,
slk::Builder *builder) { slk::Builder *builder) {
slk::Save(symbols.size(), builder); slk::Save(symbols.size(), builder);

3
tests/unit/CMakeLists.txt
View File

@ -226,9 +226,6 @@ target_link_libraries(${test_prefix}queue mg-single-node kvstore_dummy_lib)
add_unit_test(record_edge_vertex_accessor.cpp) add_unit_test(record_edge_vertex_accessor.cpp)
target_link_libraries(${test_prefix}record_edge_vertex_accessor mg-single-node kvstore_dummy_lib) target_link_libraries(${test_prefix}record_edge_vertex_accessor mg-single-node kvstore_dummy_lib)
add_unit_test(serialization.cpp)
target_link_libraries(${test_prefix}serialization mg-rpc)
add_unit_test(skip_list.cpp) add_unit_test(skip_list.cpp)
target_link_libraries(${test_prefix}skip_list mg-utils) target_link_libraries(${test_prefix}skip_list mg-utils)

41
tests/unit/ast_serialization.cpp
View File

@ -6,7 +6,6 @@
#include <vector> #include <vector>
#include <antlr4-runtime.h> #include <antlr4-runtime.h>
#include <capnp/message.h>
#include <gmock/gmock.h> #include <gmock/gmock.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
@ -74,43 +73,6 @@ class Base {
} }
}; };
class CapnpAstGenerator : public Base {
public:
Query *ParseQuery(const std::string &query_string) override {
::frontend::opencypher::Parser parser(query_string);
AstStorage tmp_storage;
CypherMainVisitor visitor(context_, &tmp_storage);
visitor.visit(parser.tree());
::capnp::MallocMessageBuilder message;
{
query::capnp::Tree::Builder builder =
message.initRoot<query::capnp::Tree>();
Save(*visitor.query(), &builder);
}
{
const query::capnp::Tree::Reader reader =
message.getRoot<query::capnp::Tree>();
return dynamic_cast<Query *>(Load(&storage_, reader));
}
}
PropertyIx Prop(const std::string &prop_name) override {
return storage_.GetPropertyIx(prop_name);
}
LabelIx Label(const std::string &name) override {
return storage_.GetLabelIx(name);
}
EdgeTypeIx EdgeType(const std::string &name) override {
return storage_.GetEdgeTypeIx(name);
}
AstStorage storage_;
};
class SlkAstGenerator : public Base { class SlkAstGenerator : public Base {
public: public:
Query *ParseQuery(const std::string &query_string) override { Query *ParseQuery(const std::string &query_string) override {
@ -148,7 +110,6 @@ class CypherMainVisitorTest
: public ::testing::TestWithParam<std::shared_ptr<Base>> {}; : public ::testing::TestWithParam<std::shared_ptr<Base>> {};
std::shared_ptr<Base> gAstGeneratorTypes[] = { std::shared_ptr<Base> gAstGeneratorTypes[] = {
std::make_shared<CapnpAstGenerator>(),
std::make_shared<SlkAstGenerator>(), std::make_shared<SlkAstGenerator>(),
}; };
@ -163,7 +124,7 @@ INSTANTIATE_TEST_CASE_P(AstGeneratorTypes, CypherMainVisitorTest,
// future reference in case someone gets the idea to change to *appropriate* // future reference in case someone gets the idea to change to *appropriate*
// Typed Tests mechanism and ruin the compilation times. // Typed Tests mechanism and ruin the compilation times.
// //
// typedef ::testing::Types<CapnpAstGenerator, SlkAstGenerator> // typedef ::testing::Types<SlkAstGenerator>
// AstGeneratorTypes; // AstGeneratorTypes;
// //
// TYPED_TEST_CASE(CypherMainVisitorTest, AstGeneratorTypes); // TYPED_TEST_CASE(CypherMainVisitorTest, AstGeneratorTypes);

149
tests/unit/distributed_query_plan.cpp
View File

@ -574,47 +574,6 @@ class ExpectPullRemoteOrderBy : public OpChecker<PullRemoteOrderBy> {
std::vector<Symbol> symbols_; std::vector<Symbol> symbols_;
}; };
void SavePlan(const LogicalOperator &plan, ::capnp::MessageBuilder *message) {
auto builder = message->initRoot<query::plan::capnp::LogicalOperator>();
LogicalOperator::SaveHelper helper;
Save(plan, &builder, &helper);
}
auto LoadPlan(const ::query::plan::capnp::LogicalOperator::Reader &reader) {
std::unique_ptr<LogicalOperator> plan;
LogicalOperator::LoadHelper helper;
Load(&plan, reader, &helper);
return std::make_pair(std::move(plan), std::move(helper.ast_storage));
}
class CapnpPlanner {
public:
template <class TDbAccessor>
CapnpPlanner(std::vector<SingleQueryPart> single_query_parts,
PlanningContext<TDbAccessor> context) {
::capnp::MallocMessageBuilder message;
{
query::Parameters parameters;
PostProcessor post_processor(parameters);
auto original_plan = MakeLogicalPlanForSingleQuery<RuleBasedPlanner>(
single_query_parts, &context);
original_plan =
post_processor.Rewrite(std::move(original_plan), &context);
SavePlan(*original_plan, &message);
}
{
auto reader = message.getRoot<query::plan::capnp::LogicalOperator>();
std::tie(plan_, ast_storage_) = LoadPlan(reader);
}
}
auto &plan() { return *plan_; }
private:
AstStorage ast_storage_;
std::unique_ptr<LogicalOperator> plan_;
};
struct ExpectedDistributedPlan { struct ExpectedDistributedPlan {
std::list<std::unique_ptr<BaseOpChecker>> master_checkers; std::list<std::unique_ptr<BaseOpChecker>> master_checkers;
std::vector<std::list<std::unique_ptr<BaseOpChecker>>> worker_checkers; std::vector<std::list<std::unique_ptr<BaseOpChecker>>> worker_checkers;
@ -729,7 +688,7 @@ class Planner {
template <class T> template <class T>
class TestPlanner : public ::testing::Test {}; class TestPlanner : public ::testing::Test {};
using PlannerTypes = ::testing::Types<Planner, CapnpPlanner>; using PlannerTypes = ::testing::Types<Planner>;
TYPED_TEST_CASE(TestPlanner, PlannerTypes); TYPED_TEST_CASE(TestPlanner, PlannerTypes);
@ -2183,109 +2142,3 @@ TYPED_TEST(TestPlanner, DistributedOptionalScanExpandExisting) {
CheckDistributedPlan(storage, planner.plan(), symbol_table, properties_by_ix, CheckDistributedPlan(storage, planner.plan(), symbol_table, properties_by_ix,
expected); expected);
} }
TEST(CapnpSerial, Union) {
std::vector<Symbol> left_symbols{
Symbol("symbol", 1, true, Symbol::Type::EDGE)};
std::vector<Symbol> right_symbols{
Symbol("symbol", 3, true, Symbol::Type::ANY)};
auto union_symbols = right_symbols;
auto union_op = std::make_unique<Union>(nullptr, nullptr, union_symbols,
left_symbols, right_symbols);
std::unique_ptr<LogicalOperator> loaded_plan;
::capnp::MallocMessageBuilder message;
SavePlan(*union_op, &message);
AstStorage new_storage;
std::tie(loaded_plan, new_storage) =
LoadPlan(message.getRoot<query::plan::capnp::LogicalOperator>());
ASSERT_TRUE(loaded_plan);
auto *loaded_op = dynamic_cast<Union *>(loaded_plan.get());
ASSERT_TRUE(loaded_op);
EXPECT_FALSE(loaded_op->left_op_);
EXPECT_FALSE(loaded_op->right_op_);
EXPECT_EQ(loaded_op->left_symbols_, left_symbols);
EXPECT_EQ(loaded_op->right_symbols_, right_symbols);
EXPECT_EQ(loaded_op->union_symbols_, union_symbols);
}
TEST(CapnpSerial, Cartesian) {
std::vector<Symbol> left_symbols{
Symbol("left_symbol", 1, true, Symbol::Type::EDGE)};
std::vector<Symbol> right_symbols{
Symbol("right_symbol", 3, true, Symbol::Type::ANY)};
auto cartesian = std::make_unique<Cartesian>(nullptr, left_symbols, nullptr,
right_symbols);
std::unique_ptr<LogicalOperator> loaded_plan;
::capnp::MallocMessageBuilder message;
SavePlan(*cartesian, &message);
AstStorage new_storage;
std::tie(loaded_plan, new_storage) =
LoadPlan(message.getRoot<query::plan::capnp::LogicalOperator>());
ASSERT_TRUE(loaded_plan);
auto *loaded_op = dynamic_cast<Cartesian *>(loaded_plan.get());
ASSERT_TRUE(loaded_op);
EXPECT_FALSE(loaded_op->left_op_);
EXPECT_FALSE(loaded_op->right_op_);
EXPECT_EQ(loaded_op->left_symbols_, left_symbols);
EXPECT_EQ(loaded_op->right_symbols_, right_symbols);
}
TEST(CapnpSerial, Synchronize) {
auto synchronize = std::make_unique<Synchronize>(nullptr, nullptr, true);
std::unique_ptr<LogicalOperator> loaded_plan;
::capnp::MallocMessageBuilder message;
SavePlan(*synchronize, &message);
AstStorage new_storage;
std::tie(loaded_plan, new_storage) =
LoadPlan(message.getRoot<query::plan::capnp::LogicalOperator>());
ASSERT_TRUE(loaded_plan);
auto *loaded_op = dynamic_cast<Synchronize *>(loaded_plan.get());
ASSERT_TRUE(loaded_op);
EXPECT_FALSE(loaded_op->input());
EXPECT_FALSE(loaded_op->pull_remote_);
EXPECT_TRUE(loaded_op->advance_command_);
}
TEST(CapnpSerial, PullRemote) {
std::vector<Symbol> symbols{Symbol("symbol", 1, true, Symbol::Type::EDGE)};
auto pull_remote = std::make_unique<PullRemote>(nullptr, 42, symbols);
std::unique_ptr<LogicalOperator> loaded_plan;
::capnp::MallocMessageBuilder message;
SavePlan(*pull_remote, &message);
AstStorage new_storage;
std::tie(loaded_plan, new_storage) =
LoadPlan(message.getRoot<query::plan::capnp::LogicalOperator>());
ASSERT_TRUE(loaded_plan);
auto *loaded_op = dynamic_cast<PullRemote *>(loaded_plan.get());
ASSERT_TRUE(loaded_op);
EXPECT_FALSE(loaded_op->input());
EXPECT_EQ(loaded_op->plan_id_, 42);
EXPECT_EQ(loaded_op->symbols_, symbols);
}
TEST(CapnpSerial, PullRemoteOrderBy) {
auto once = std::make_shared<Once>();
AstStorage storage;
std::vector<Symbol> symbols{
Symbol("my_symbol", 2, true, Symbol::Type::VERTEX, 3)};
std::vector<query::SortItem> order_by{
{query::Ordering::ASC, IDENT("my_symbol")}};
auto pull_remote_order_by =
std::make_unique<PullRemoteOrderBy>(once, 42, order_by, symbols);
std::unique_ptr<LogicalOperator> loaded_plan;
::capnp::MallocMessageBuilder message;
SavePlan(*pull_remote_order_by, &message);
AstStorage new_storage;
std::tie(loaded_plan, new_storage) =
LoadPlan(message.getRoot<query::plan::capnp::LogicalOperator>());
ASSERT_TRUE(loaded_plan);
auto *loaded_op = dynamic_cast<PullRemoteOrderBy *>(loaded_plan.get());
ASSERT_TRUE(loaded_op);
ASSERT_TRUE(std::dynamic_pointer_cast<Once>(loaded_op->input()));
EXPECT_EQ(loaded_op->plan_id_, 42);
EXPECT_EQ(loaded_op->symbols_, symbols);
ASSERT_EQ(loaded_op->order_by_.size(), 1);
EXPECT_TRUE(dynamic_cast<query::Identifier *>(loaded_op->order_by_[0]));
ASSERT_EQ(loaded_op->compare_.ordering().size(), 1);
EXPECT_EQ(loaded_op->compare_.ordering()[0], query::Ordering::ASC);
}

1
tests/unit/rpc.cpp
View File

@ -1,6 +1,5 @@
#include <thread> #include <thread>
#include "capnp/serialize.h"
#include "gmock/gmock.h" #include "gmock/gmock.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"

334
tests/unit/serialization.cpp
View File

@ -1,334 +0,0 @@
#include <optional>
#include <sstream>
#include "gtest/gtest.h"
#include "capnp/message.h"
#include "rpc/serialization.hpp"
using std::optional;
using std::string_literals::operator""s;
void CheckOptionalInt(const std::optional<int> &x1) {
::capnp::MallocMessageBuilder message;
std::optional<int> y1;
{
auto builder =
message.initRoot<utils::capnp::Optional<utils::capnp::BoxInt32>>();
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveOptional<utils::capnp::BoxInt32, int>(x1, &builder, save);
}
{
auto reader =
message.getRoot<utils::capnp::Optional<utils::capnp::BoxInt32>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
y1 = utils::LoadOptional<utils::capnp::BoxInt32, int>(reader, load);
}
EXPECT_EQ(x1, y1);
}
TEST(Serialization, CapnpOptional) {
std::optional<int> x1 = {};
std::optional<int> x2 = 42;
CheckOptionalInt(x1);
CheckOptionalInt(x2);
}
TEST(Serialization, CapnpOptionalNonCopyable) {
std::optional<std::unique_ptr<int>> data = std::make_unique<int>(5);
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::Optional<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](auto *ptr_builder, const auto &data) {
auto save_int = [](auto *int_builder, int value) {
int_builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, ptr_builder,
save_int);
};
utils::SaveOptional<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save);
}
std::optional<std::unique_ptr<int>> element;
{
auto reader = message.getRoot<utils::capnp::Optional<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](const auto &ptr_reader) {
auto load_int = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(ptr_reader,
load_int);
};
element =
utils::LoadOptional<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load);
}
EXPECT_EQ(*element.value(), 5);
}
void CheckUniquePtrInt(const std::unique_ptr<int> &x1) {
::capnp::MallocMessageBuilder message;
std::unique_ptr<int> y1;
{
auto builder =
message.initRoot<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>();
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(x1, &builder, save);
}
{
auto reader =
message.getRoot<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>();
auto load = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
y1 = utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(reader, load);
}
if (!x1)
EXPECT_EQ(y1, nullptr);
else
EXPECT_EQ(*x1, *y1);
}
TEST(Serialization, CapnpUniquePtr) {
auto x1 = std::make_unique<int>(42);
std::unique_ptr<int> x2;
CheckUniquePtrInt(x1);
CheckUniquePtrInt(x2);
}
TEST(Serialization, CapnpUniquePtrNonCopyable) {
std::unique_ptr<std::unique_ptr<int>> data =
std::make_unique<std::unique_ptr<int>>(std::make_unique<int>(5));
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::UniquePtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](auto *ptr_builder, const auto &data) {
auto save_int = [](auto *int_builder, int value) {
int_builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, ptr_builder,
save_int);
};
utils::SaveUniquePtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save);
}
std::unique_ptr<std::unique_ptr<int>> element;
{
auto reader = message.getRoot<utils::capnp::UniquePtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](const auto &ptr_reader) {
auto load_int = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
return new std::unique_ptr<int>(
utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(ptr_reader,
load_int));
};
element =
utils::LoadUniquePtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load);
}
EXPECT_EQ(**element, 5);
}
TEST(Serialization, CapnpSharedPtr) {
std::vector<int *> saved_pointers;
auto p1 = std::make_shared<int>(5);
std::shared_ptr<int> p2;
std::vector<std::shared_ptr<int>> pointers{p1, p1, p2};
::capnp::MallocMessageBuilder message;
{
auto builders = message.initRoot<
::capnp::List<utils::capnp::SharedPtr<utils::capnp::BoxInt32>>>(
pointers.size());
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
for (size_t i = 0; i < pointers.size(); ++i) {
auto ptr_builder = builders[i];
utils::SaveSharedPtr<utils::capnp::BoxInt32, int>(
pointers[i], &ptr_builder, save, &saved_pointers);
}
}
EXPECT_EQ(saved_pointers.size(), 1);
std::vector<std::pair<uint64_t, std::shared_ptr<int>>> loaded_pointers;
std::vector<std::shared_ptr<int>> elements;
{
auto reader = message.getRoot<
::capnp::List<utils::capnp::SharedPtr<utils::capnp::BoxInt32>>>();
auto load = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
for (const auto ptr_reader : reader) {
elements.emplace_back(utils::LoadSharedPtr<utils::capnp::BoxInt32, int>(
ptr_reader, load, &loaded_pointers));
}
}
EXPECT_EQ(loaded_pointers.size(), 1);
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(*elements[0], 5);
EXPECT_EQ(*elements[0], *elements[1]);
EXPECT_EQ(elements[2].get(), nullptr);
}
TEST(Serialization, CapnpSharedPtrNonCopyable) {
std::shared_ptr<std::unique_ptr<int>> data =
std::make_shared<std::unique_ptr<int>>(std::make_unique<int>(5));
std::vector<std::unique_ptr<int> *> saved_pointers;
::capnp::MallocMessageBuilder message;
{
auto builder = message.initRoot<utils::capnp::SharedPtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto save = [](auto *ptr_builder, const auto &data) {
auto save_int = [](auto *int_builder, int value) {
int_builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, ptr_builder,
save_int);
};
utils::SaveSharedPtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &builder, save,
&saved_pointers);
}
std::shared_ptr<std::unique_ptr<int>> element;
std::vector<std::pair<uint64_t, std::shared_ptr<std::unique_ptr<int>>>>
loaded_pointers;
{
auto reader = message.getRoot<utils::capnp::SharedPtr<
utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](const auto &ptr_reader) {
auto load_int = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
return new std::unique_ptr<int>(
utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(ptr_reader,
load_int));
};
element =
utils::LoadSharedPtr<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(reader, load,
&loaded_pointers);
}
EXPECT_EQ(**element, 5);
}
TEST(Serialization, CapnpVectorPrimitive) {
std::vector<int> data{1, 2, 3};
::capnp::MallocMessageBuilder message;
{
auto list_builder = message.initRoot<::capnp::List<int>>(data.size());
utils::SaveVector<int>(data, &list_builder);
}
std::vector<int> elements;
{
auto reader = message.getRoot<::capnp::List<int>>();
utils::LoadVector<int>(&elements, reader);
}
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(elements[0], 1);
EXPECT_EQ(elements[1], 2);
EXPECT_EQ(elements[2], 3);
}
TEST(Serialization, CapnpVector) {
std::vector<int> data{1, 2, 3};
::capnp::MallocMessageBuilder message;
{
auto list_builder =
message.initRoot<::capnp::List<utils::capnp::BoxInt32>>(data.size());
auto save = [](utils::capnp::BoxInt32::Builder *builder, int value) {
builder->setValue(value);
};
utils::SaveVector<utils::capnp::BoxInt32, int>(data, &list_builder, save);
}
std::vector<int> elements;
{
auto reader = message.getRoot<::capnp::List<utils::capnp::BoxInt32>>();
auto load = [](const utils::capnp::BoxInt32::Reader &reader) -> int {
return reader.getValue();
};
utils::LoadVector<utils::capnp::BoxInt32, int>(&elements, reader, load);
}
EXPECT_EQ(elements.size(), 3);
EXPECT_EQ(elements[0], 1);
EXPECT_EQ(elements[1], 2);
EXPECT_EQ(elements[2], 3);
}
TEST(Serialization, CapnpVectorNonCopyable) {
std::vector<std::unique_ptr<int>> data;
data.emplace_back(std::make_unique<int>(5));
data.emplace_back(std::make_unique<int>(10));
::capnp::MallocMessageBuilder message;
{
auto list_builder = message.initRoot<
::capnp::List<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>(
data.size());
auto save = [](auto *ptr_builder, const auto &data) {
auto save_int = [](auto *int_builder, int value) {
int_builder->setValue(value);
};
utils::SaveUniquePtr<utils::capnp::BoxInt32, int>(data, ptr_builder,
save_int);
};
utils::SaveVector<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(data, &list_builder, save);
}
std::vector<std::unique_ptr<int>> elements;
{
auto reader = message.getRoot<
::capnp::List<utils::capnp::UniquePtr<utils::capnp::BoxInt32>>>();
auto load = [](const auto &ptr_reader) {
auto load_int = [](const auto &int_reader) {
return new int(int_reader.getValue());
};
return utils::LoadUniquePtr<utils::capnp::BoxInt32, int>(ptr_reader,
load_int);
};
utils::LoadVector<utils::capnp::UniquePtr<utils::capnp::BoxInt32>,
std::unique_ptr<int>>(&elements, reader, load);
}
EXPECT_EQ(elements.size(), 2);
EXPECT_EQ(*elements[0], 5);
EXPECT_EQ(*elements[1], 10);
}
TEST(Serialization, CapnpMap) {
std::map<std::string, std::string> map{{"my_key", "my_value"},
{"other_key", "other_value"}};
::capnp::MallocMessageBuilder message;
{
auto map_builder =
message.initRoot<utils::capnp::Map<capnp::Text, capnp::Text>>();
utils::SaveMap<capnp::Text, capnp::Text>(
map, &map_builder, [](auto *entry_builder, const auto &entry) {
entry_builder->setKey(entry.first);
entry_builder->setValue(entry.second);
});
}
std::map<std::string, std::string> new_map;
{
auto map_reader =
message.getRoot<utils::capnp::Map<capnp::Text, capnp::Text>>();
utils::LoadMap<capnp::Text, capnp::Text>(
&new_map, map_reader, [](const auto &entry_reader) {
std::string key = entry_reader.getKey();
std::string value = entry_reader.getValue();
return std::make_pair(key, value);
});
}
EXPECT_EQ(new_map, map);
}

12
tools/lcp
View File

@ -1,9 +1,9 @@
#!/bin/bash -e #!/bin/bash -e
if [[ $# -ne 1 && $# -ne 2 ]]; then if [[ $# -ne 1 && $# -ne 2 ]]; then
echo "Usage: $0 LCP_FILE [CAPNP_ID]" echo "Usage: $0 LCP_FILE"
echo "Convert a LCP_FILE to C++ header file and output to stdout." echo "Convert a LCP_FILE to C++ header file and output to stdout."
echo "If CAPNP_ID is provided, then the Cap'n Proto schema is generated." echo "If SLK_SERIALIZE is provided, then SLK serialization is generated."
exit 1 exit 1
fi fi
@ -20,16 +20,16 @@ if [[ -v QUICKLISP_HOME ]]; then
quicklisp_install_dir="${QUICKLISP_HOME}" quicklisp_install_dir="${QUICKLISP_HOME}"
fi fi
capnp="" slk_serialize=""
if [[ $# -eq 2 ]]; then if [[ "$2" == "SLK_SERIALIZE" ]]; then
capnp=":capnp-id \"$2\"" slk_serialize=":slk-serialize t"
fi fi
echo \ echo \
" "
(load \"${quicklisp_install_dir}/setup.lisp\") (load \"${quicklisp_install_dir}/setup.lisp\")
(ql:quickload :lcp :silent t) (ql:quickload :lcp :silent t)
(lcp:process-file \"$lcp_file\" $capnp) (lcp:process-file \"$lcp_file\" $slk_serialize)
" | sbcl --script " | sbcl --script
filename=`basename $lcp_file .lcp` filename=`basename $lcp_file .lcp`