diff --git a/cmake/FindSodium.cmake b/cmake/FindSodium.cmake new file mode 100644 index 000000000..dd1398d73 --- /dev/null +++ b/cmake/FindSodium.cmake @@ -0,0 +1,303 @@ +# Written in 2016 by Henrik Steffen Gaßmann <henrik@gassmann.onl> +# +# To the extent possible under law, the author(s) have dedicated all +# copyright and related and neighboring rights to this software to the +# public domain worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication +# along with this software. If not, see +# +# http://creativecommons.org/publicdomain/zero/1.0/ +# +# ####################################################################### +# Tries to find the local libsodium installation. +# +# On Windows the sodium_DIR environment variable is used as a default +# hint which can be overridden by setting the corresponding cmake variable. +# +# Once done the following variables will be defined: +# +# sodium_FOUND +# sodium_INCLUDE_DIR +# sodium_LIBRARY_DEBUG +# sodium_LIBRARY_RELEASE +# +# +# Furthermore an imported "sodium" target is created. +# + +if(CMAKE_C_COMPILER_ID STREQUAL "GNU" + OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(_GCC_COMPATIBLE 1) +endif() + +# static library option +if(NOT DEFINED sodium_USE_STATIC_LIBS) + option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF) +endif() + +if(NOT(sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST)) + unset(sodium_LIBRARY CACHE) + unset(sodium_LIBRARY_DEBUG CACHE) + unset(sodium_LIBRARY_RELEASE CACHE) + unset(sodium_DLL_DEBUG CACHE) + unset(sodium_DLL_RELEASE CACHE) + set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") +endif() + +# ####################################################################### +# UNIX +if(UNIX) + # import pkg-config + find_package(PkgConfig QUIET) + + if(PKG_CONFIG_FOUND) + pkg_check_modules(sodium_PKG QUIET libsodium) + endif() + + if(sodium_USE_STATIC_LIBS) + foreach(_libname ${sodium_PKG_STATIC_LIBRARIES}) + if(NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a + list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a") + endif() + endforeach() + + list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES) + + # if pkgconfig for libsodium doesn't provide + # static lib info, then override PKG_STATIC here.. + if(NOT sodium_PKG_STATIC_FOUND) + set(sodium_PKG_STATIC_LIBRARIES libsodium.a) + endif() + + set(XPREFIX sodium_PKG_STATIC) + else() + if(NOT sodium_PKG_FOUND) + set(sodium_PKG_LIBRARIES sodium) + endif() + + set(XPREFIX sodium_PKG) + endif() + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${${XPREFIX}_INCLUDE_DIRS} + ) + find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + +# ####################################################################### +# Windows +elseif(WIN32) + set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory") + mark_as_advanced(sodium_DIR) + + find_path(sodium_INCLUDE_DIR sodium.h + HINTS ${sodium_DIR} + PATH_SUFFIXES include + ) + + if(MSVC) + # detect target architecture + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[ + #if defined _M_IX86 + #error ARCH_VALUE x86_32 + #elif defined _M_X64 + #error ARCH_VALUE x86_64 + #endif + #error ARCH_VALUE unknown + ]=]) + try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" + OUTPUT_VARIABLE _COMPILATION_LOG + ) + string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}") + + # construct library path + if(_TARGET_ARCH STREQUAL "x86_32") + string(APPEND _PLATFORM_PATH "Win32") + elseif(_TARGET_ARCH STREQUAL "x86_64") + string(APPEND _PLATFORM_PATH "x64") + else() + message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.") + endif() + + string(APPEND _PLATFORM_PATH "/$$CONFIG$$") + + if(MSVC_VERSION LESS 1900) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") + else() + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + endif() + + string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") + + if(sodium_USE_STATIC_LIBS) + string(APPEND _PLATFORM_PATH "/static") + else() + string(APPEND _PLATFORM_PATH "/dynamic") + endif() + + string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}") + string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}") + + find_library(sodium_LIBRARY_DEBUG libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_LIBRARY_RELEASE libsodium.lib + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + + if(NOT sodium_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll") + find_library(sodium_DLL_DEBUG libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_DLL_RELEASE libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) + endif() + + elseif(_GCC_COMPATIBLE) + if(sodium_USE_STATIC_LIBS) + find_library(sodium_LIBRARY_DEBUG libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + else() + find_library(sodium_LIBRARY_DEBUG libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + find_library(sodium_LIBRARY_RELEASE libsodium.dll.a + HINTS ${sodium_DIR} + PATH_SUFFIXES lib + ) + + file(GLOB _DLL + LIST_DIRECTORIES false + RELATIVE "${sodium_DIR}/bin" + "${sodium_DIR}/bin/libsodium*.dll" + ) + find_library(sodium_DLL_DEBUG ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + find_library(sodium_DLL_RELEASE ${_DLL} libsodium + HINTS ${sodium_DIR} + PATH_SUFFIXES bin + ) + endif() + else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") + endif() + +# ####################################################################### +# unsupported +else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") +endif() + +# ####################################################################### +# common stuff + +# extract sodium version +if(sodium_INCLUDE_DIR) + set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h") + + if(EXISTS _VERSION_HEADER) + file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT) + string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1" + sodium_VERSION "${_VERSION_HEADER_CONTENT}") + set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE) + endif() +endif() + +# communicate results +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Sodium # The name must be either uppercase or match the filename case. + REQUIRED_VARS + sodium_LIBRARY_RELEASE + sodium_LIBRARY_DEBUG + sodium_INCLUDE_DIR + VERSION_VAR + sodium_VERSION +) + +if(Sodium_FOUND) + set(sodium_LIBRARIES + optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG}) +endif() + +# mark file paths as advanced +mark_as_advanced(sodium_INCLUDE_DIR) +mark_as_advanced(sodium_LIBRARY_DEBUG) +mark_as_advanced(sodium_LIBRARY_RELEASE) + +if(WIN32) + mark_as_advanced(sodium_DLL_DEBUG) + mark_as_advanced(sodium_DLL_RELEASE) +endif() + +# create imported target +if(sodium_USE_STATIC_LIBS) + set(_LIB_TYPE STATIC) +else() + set(_LIB_TYPE SHARED) +endif() + +if(NOT TARGET sodium) + add_library(sodium ${_LIB_TYPE} IMPORTED) +endif() + +set_target_properties(sodium PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" +) + +if(sodium_USE_STATIC_LIBS) + set_target_properties(sodium PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) +else() + if(UNIX) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + elseif(WIN32) + set_target_properties(sodium PROPERTIES + IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}" + IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}" + ) + + if(NOT(sodium_DLL_DEBUG MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}" + ) + endif() + + if(NOT(sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}" + ) + endif() + endif() +endif() diff --git a/cmake/MgThrift.cmake b/cmake/MgThrift.cmake new file mode 100644 index 000000000..779cd3c61 --- /dev/null +++ b/cmake/MgThrift.cmake @@ -0,0 +1,176 @@ +find_package(Gflags) +find_package(Folly) +find_package(wangle) +find_package(fizz) +find_package(proxygen) +find_package(FBThrift) +set(THRIFTCPP2 "FBThrift::thriftcpp2") +set(THRIFT1 ${FBTHRIFT_COMPILER}) + +include(${FBTHRIFT_INCLUDE_DIR}/thrift/ThriftLibrary.cmake) + +set(MG_INTERFACE_TARGET_NAME_PREFIX "mg-interface") + +function(_mg_thrift_generate + file_name + services + language + options + file_path + output_path + include_prefix +) + cmake_parse_arguments(THRIFT_GENERATE # Prefix + "" # Options + "" # One Value args + "THRIFT_INCLUDE_DIRECTORIES" # Multi-value args + "${ARGN}") + + set(thrift_include_directories) + + foreach(dir ${THRIFT_GENERATE_THRIFT_INCLUDE_DIRECTORIES}) + list(APPEND thrift_include_directories "-I" "${dir}") + endforeach() + + set("${file_name}-${language}-HEADERS" + ${output_path}/gen-${language}/${file_name}_constants.h + ${output_path}/gen-${language}/${file_name}_data.h + ${output_path}/gen-${language}/${file_name}_metadata.h + ${output_path}/gen-${language}/${file_name}_types.h + ${output_path}/gen-${language}/${file_name}_types.tcc + ) + set("${file_name}-${language}-SOURCES" + ${output_path}/gen-${language}/${file_name}_constants.cpp + ${output_path}/gen-${language}/${file_name}_data.cpp + ${output_path}/gen-${language}/${file_name}_types.cpp + ) + + if(NOT "${options}" MATCHES "no_metadata") + set("${file_name}-${language}-SOURCES" + ${${file_name}-${language}-SOURCES} + ${output_path}/gen-${language}/${file_name}_metadata.cpp + ) + endif() + + foreach(service ${services}) + set("${file_name}-${language}-HEADERS" + ${${file_name}-${language}-HEADERS} + ${output_path}/gen-${language}/${service}.h + ${output_path}/gen-${language}/${service}.tcc + ${output_path}/gen-${language}/${service}AsyncClient.h + ${output_path}/gen-${language}/${service}_custom_protocol.h + ) + set("${file_name}-${language}-SOURCES" + ${${file_name}-${language}-SOURCES} + ${output_path}/gen-${language}/${service}.cpp + ${output_path}/gen-${language}/${service}AsyncClient.cpp + ) + endforeach() + + if("${include_prefix}" STREQUAL "") + set(include_prefix_text "") + else() + set(include_prefix_text "include_prefix=${include_prefix}") + + if(NOT "${options}" STREQUAL "") + set(include_prefix_text ",${include_prefix_text}") + endif() + endif() + + set(gen_language ${language}) + + if("${language}" STREQUAL "cpp2") + set(gen_language "mstch_cpp2") + elseif("${language}" STREQUAL "py3") + set(gen_language "mstch_py3") + file(WRITE "${output_path}/gen-${language}/${file_name}/__init__.py") + endif() + + add_custom_command( + OUTPUT ${${file_name}-${language}-HEADERS} + ${${file_name}-${language}-SOURCES} + COMMAND ${THRIFT1} + --gen "${gen_language}:${options}${include_prefix_text}" + -o ${output_path} + ${thrift_include_directories} + "${file_path}/${file_name}.thrift" + DEPENDS + ${THRIFT1} + "${file_path}/${file_name}.thrift" + COMMENT "Generating ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language} files. Output: ${output_path}" + ) + add_custom_target( + ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target ALL + DEPENDS ${${language}-${language}-HEADERS} + ${${file_name}-${language}-SOURCES} + ) + + set("${file_name}-${language}-SOURCES" ${${file_name}-${language}-SOURCES} PARENT_SCOPE) + install( + DIRECTORY gen-${language} + DESTINATION include/${include_prefix} + FILES_MATCHING PATTERN "*.h") + install( + DIRECTORY gen-${language} + DESTINATION include/${include_prefix} + FILES_MATCHING PATTERN "*.tcc") +endfunction() + +function(_mg_thrift_object + file_name + services + language + options + file_path + output_path + include_prefix +) + _mg_thrift_generate( + "${file_name}" + "${services}" + "${language}" + "${options}" + "${file_path}" + "${output_path}" + "${include_prefix}" + "${ARGN}" + ) + bypass_source_check(${${file_name}-${language}-SOURCES}) + add_library( + "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj" + OBJECT + ${${file_name}-${language}-SOURCES} + ) + add_dependencies( + "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj" + "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target" + ) + target_include_directories(${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj PUBLIC ${output_path}) + message(STATUS "MgThrift will create the Object file : ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj") +endfunction() + +function(mg_thrift_library + file_name + services + file_path + output_path + include_prefix +) + _mg_thrift_object( + "${file_name}" + "${services}" + "cpp2" + "stack_arguments" # options + "${file_path}" + "${output_path}" + "${include_prefix}" + THRIFT_INCLUDE_DIRECTORIES "${FBTHRIFT_INCLUDE_DIR}" + ) + set(LIBRARY_NAME "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-cpp2") + add_library( + "${LIBRARY_NAME}" + $<TARGET_OBJECTS:${LIBRARY_NAME}-obj> + ) + target_link_libraries("${LIBRARY_NAME}" ${THRIFTCPP2}) + message("MgThrift will create the library file : ${LIBRARY_NAME}") +endfunction() diff --git a/environment/toolchain/fbthrift.patch b/environment/toolchain/fbthrift.patch new file mode 100644 index 000000000..c1b370247 --- /dev/null +++ b/environment/toolchain/fbthrift.patch @@ -0,0 +1,12 @@ +diff -ur a/thrift/lib/cpp2/server/IOWorkerContext.h b/thrift/lib/cpp2/server/IOWorkerContext.h +--- a/thrift/lib/cpp2/server/IOWorkerContext.h 2022-06-08 11:50:43.043948657 +0200 ++++ b/thrift/lib/cpp2/server/IOWorkerContext.h 2022-06-08 11:47:33.232695125 +0200 +@@ -59,7 +59,7 @@ + auto aliveLocked = alive->rlock(); + if (*aliveLocked) { + // IOWorkerContext is still alive and so is replyQueue_ +- queue->startConsumingInternal(&evb); ++ queue->startConsuming(&evb); + } + }); + } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a510aee6..3a239dda2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,7 @@ # CMake configuration for the main memgraph library and executable # add memgraph sub libraries, ordered by dependency +add_subdirectory(interface) add_subdirectory(lisp) add_subdirectory(utils) add_subdirectory(requests) diff --git a/src/interface/.gitignore b/src/interface/.gitignore new file mode 100644 index 000000000..38674d178 --- /dev/null +++ b/src/interface/.gitignore @@ -0,0 +1 @@ +gen-cpp2 diff --git a/src/interface/CMakeLists.txt b/src/interface/CMakeLists.txt new file mode 100644 index 000000000..8cbdf3433 --- /dev/null +++ b/src/interface/CMakeLists.txt @@ -0,0 +1,35 @@ +include(MgThrift) + +set(MG_INTERFACE_PATH ${CMAKE_CURRENT_SOURCE_DIR}) + +mg_thrift_library( + "storage" #file_name + "Storage" #services + "${MG_INTERFACE_PATH}" #file_path + "${MG_INTERFACE_PATH}" #output_path + "interface" #include_prefix +) + +mg_thrift_library( + "meta" #file_name + "Meta" #services + "${MG_INTERFACE_PATH}" #file_path + "${MG_INTERFACE_PATH}" #output_path + "interface" #include_prefix +) + +mg_thrift_library( + "trial" #file_name + "PingPong" #services + "${MG_INTERFACE_PATH}" #file_path + "${MG_INTERFACE_PATH}" #output_path + "interface" #include_prefix +) + +mg_thrift_library( + "echo" #file_name + "Echo" #services + "${MG_INTERFACE_PATH}" #file_path + "${MG_INTERFACE_PATH}" #output_path + "interface" #include_prefix +) diff --git a/src/interface/counter.thrift b/src/interface/counter.thrift new file mode 100644 index 000000000..3460264d8 --- /dev/null +++ b/src/interface/counter.thrift @@ -0,0 +1,22 @@ +namespace cpp2 interface.counter +// https://stackoverflow.com/a/34234874/6639989 + +struct GetLatestReqeuest { + 1: optional i64 proposed_value; +} + +struct GetLatestResponse { + 1: i64 value; +} + +union CounterRequest { + 1: GetLatestReqeuest get_latest; +} + +union CounterResponse { + 1: GetLatestResponse get_latest; +} + +service Counter { + CounterResponse Request(1: CounterRequest req) +} diff --git a/src/interface/echo.thrift b/src/interface/echo.thrift new file mode 100644 index 000000000..b7059f828 --- /dev/null +++ b/src/interface/echo.thrift @@ -0,0 +1,7 @@ +struct EchoMessage { + 1: binary message; +} + +service Echo { + oneway void ReceiveSend(1: EchoMessage m) +} diff --git a/src/interface/echo_trial.thrift b/src/interface/echo_trial.thrift new file mode 100644 index 000000000..b7059f828 --- /dev/null +++ b/src/interface/echo_trial.thrift @@ -0,0 +1,7 @@ +struct EchoMessage { + 1: binary message; +} + +service Echo { + oneway void ReceiveSend(1: EchoMessage m) +} diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift new file mode 100644 index 000000000..b2384d843 --- /dev/null +++ b/src/interface/meta.thrift @@ -0,0 +1,74 @@ +namespace cpp2 interface.meta + +typedef i64 LabelId +typedef i64 IndexId + +struct Result { + 1: bool success; +} + +struct CreatePrimaryLabelRequest { + 1: binary name; + 2: list<binary> primary_keys; +} + +struct CreateLabelResponse { + 1: Result result; + 2: LabelId label_id; +} + +struct GetLabelInfosRequest { + // The empty list means return all of the label infos + 1: list<binary> label_names; +} + +struct LabelInfo { + 1: binary name; + 2: LabelId label_id; + 3: list<binary> primary_keys; +} + +struct GetLabelInfoResponse { + 1: Result result; + 2: LabelInfo label_info; +} + +struct GetLabelInfosResponse { + 1: Result result; + 2: list<LabelInfo> label_infos; +} + +struct CreateIndexRequest { + 1: LabelId label_id; + 2: list<binary> property_names; +} + +struct CreateIndexResponse { + 1: Result result; + 2: IndexId index_id; +} + +struct IndexInfo { + 1: LabelId label_id; + 2: list<binary> property_names; +} + +struct GetIndexInfosResponse { + 1: Result result; + 2: list<IndexInfo> index_infos; +} + + +service Meta { + CreateLabelResponse createPrimaryLabel(1: CreatePrimaryLabelRequest req); + CreateLabelResponse createSecondaryLabel(1: binary label_name); + Result dropLabel(1: binary label_name); + GetLabelInfoResponse getLabelInfo(1:binary label_name); + GetLabelInfosResponse getLabelInfos(1: list<binary> label_names); + + CreateIndexResponse createIndex(1: CreateIndexRequest req); + // Don't have index names, and the label doesn't identify the index uniquely, therefore + Result dropIndex(1: IndexId index_id); + GetIndexInfosResponse getIndexInfos(); + GetIndexInfosResponse getIndexInfosForLabel(1: LabelId label_id); +} diff --git a/src/interface/storage.thrift b/src/interface/storage.thrift new file mode 100644 index 000000000..3a52d0b8a --- /dev/null +++ b/src/interface/storage.thrift @@ -0,0 +1,273 @@ +namespace cpp2 interface.storage +// https://stackoverflow.com/a/34234874/6639989 + +cpp_include "storage/v2/view.hpp" + +typedef i64 VertexId +typedef i64 Gid + +// TODO(antaljanosbenjamin): Use this after introducing 128 bit vertex ids +// struct VertexId { +// 1: i64 upper_half; +// 2: i64 lower_half; +// } + +struct Label { + 1: i64 id; +} + +struct EdgeType { + 1: binary name; +} + +struct EdgeId { + 1: VertexId src; + // QUESTION(antaljanosbenjamin): is it okay to have vertex based (edge id = vertex id + edge id inside vertex)? + 2: Gid gid; +} + +struct Date { + 1: i16 year; + 2: byte month; + 3: byte day; +} + +struct LocalTime { + 1: byte hour; + 2: byte minute; + 3: byte second; + 4: i16 millisecond; + 5: i16 microsecond; +} + +struct LocalDateTime { + 1: Date date; + 2: LocalTime local_time; +} + +struct Duration { + 1: i64 milliseconds; +} + +union Value { + 1: Null null_v; + 2: bool bool_v; + 3: i64 int_v; + 4: double double_v; + 5: binary string_v; + 6: list<Value> list_v; + 7: map<binary, Value> (cpp.template = "std::unordered_map") map_v (cpp2.ref_type = "unique"); + 8: Vertex vertex_v (cpp2.ref_type = "unique"); + 9: Edge edge_v (cpp2.ref_type = "unique"); + 10: Path path_v (cpp2.ref_type = "unique"); + 11: Date date_v; + 12: LocalTime local_time_v; + 13: LocalDateTime local_date_time_v; + 14: Duration duration_v; +} + +struct Null { +} + +struct Vertex { + 1: VertexId id; + // TODO(antaljanosbenjamin): Change to sperate primary and secondary labels when schema is implemented + 2: list<Label> labels; +} + +struct Edge { + 1: VertexId src; + 2: VertexId dst; + 3: EdgeType type; +} + +struct PathPart { + 1: Vertex dst; + 2: Edge edge; +} + +struct Path { + 1: Vertex src; + 2: list<PathPart> parts; +} + +struct ValuesMap { + 1: map<i64, Value> (cpp.template = "std::unordered_map") values_map; +} + +struct MappedValues { + 1: list<ValuesMap> properties; +} + +struct ListedValues { + 1: list<list<Value>> properties; +} + +union Values { + // This struct is necessary because depending on the request the response + // has two different formats: + // 1. When the request specifies the returned properties, then they are + // returned in that order, therefore no extra mapping is necessary. + // 2. When the request doesn't specify the returned properties, then all + // of the properties are returned. In this case the `mapped` field is + // used. To extract the <key,value> pairs from this struct the + // mapping of i64 -> property name has to be used. + 1: ListedValues listed; + 2: MappedValues mapped; +} + +struct Expression { + 1: binary alias; + 2: binary expression; +} + +struct Filter { + 1: binary filter_expression; +} + +enum OrderingDirection { + ASCENDING = 1; + DESCENDING = 2; +} + +struct OrderBy { + 1: Expression expression; + 2: OrderingDirection direction; +} + +struct Result { + // Just placeholder data for now + 1: bool success; +} + +enum View { + OLD = 0, + NEW = 1 +} (cpp.enum_strict, cpp.type = "memgraph::storage::View") + +struct ScanVerticesRequest { + 1: i64 transaction_id; + 2: optional i64 start_id; + // Special values are accepted: + // * __mg__id (Vertex, but without labels) + // * __mg__labels (Vertex, but without the id) + // If both of them is specified, then it will result in a single, fully populated vertex + // QUESTION(antaljanosbenjamin): Does the `__mg__labels` is necessary? What about passing the `labels` function + // as an expression? Maybe it is an optimization. For communicating the vertex id + // the Vertex struct is really handy. + 3: optional list<binary> props_to_return; + 4: list<Expression> expressions; + 5: optional i64 limit; + 6: View view; + 7: optional Filter filter; +} + +struct ScanVerticesResponse { + 1: Result result; + 2: Values values; + 3: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map; + // contains the next start_id if there is any + 4: optional VertexId next_start_id; +} + +union VertexOrEdgeIds { + 1: list<VertexId> vertex_ids; + 2: list<EdgeId> edge_ids; +} + +struct GetPropertiesRequest { + 1: i64 transaction_id; + 2: VertexOrEdgeIds vertex_or_edge_ids; + 3: list<binary> property_names; + 4: list<Expression> expressions; + 5: bool only_unique = false; + 6: optional list<OrderBy> order_by; + 7: optional i64 limit; + 8: optional Filter filter; +} + +struct GetPropertiesResponse { + 1: Values values; + 2: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map; +} + +enum EdgeDirection { + OUT = 1; + IN = 2; + BOTH = 3; +} + +struct ExpandOneRequest { + 1: i64 transaction_id; + 2: list<VertexId> src_vertices; + 3: list<EdgeType> edge_types; + 4: EdgeDirection direction; + 5: bool only_unique_neighbor_rows = false; + // The empty optional means return all of the properties, while an empty + // list means do not return any properties + // TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object + // after schema is implemented + // Special values are accepted: + // * __mg__labels + 6: optional list<binary> src_vertex_properties; + // TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object + // after schema is implemented + // Special values are accepted: + // * __mg__dst_id (Vertex, but without labels) + // * __mg__type (binary) + 7: optional list<binary> edge_properties; + // QUESTION(antaljanosbenjamin): Maybe also add possibility to expressions evaluated on the source vertex? + // List of expressions evaluated on edges + 8: list<Expression> expressions; + 9: optional list<OrderBy> order_by; + 10: optional i64 limit; + 11: optional Filter filter; +} + +struct ExpandOneResultRow { + // NOTE: This struct could be a single Values with columns something like this: + // src_vertex(Vertex), vertex_prop1(Value), vertex_prop2(Value), edges(list<Value>) + // where edges might be a list of: + // 1. list<Value> if only a defined list of edge properties are returned + // 2. map<binary, Value> if all of the edge properties are returned + // The drawback of this is currently the key of the map is always interpreted as a string in Value, not as an + // integer, which should be in case of mapped properties. + 1: Vertex src_vertex; + 2: optional Values src_vertex_properties; + 3: Values edges; +} + +struct ExpandOneResponse { + // This approach might not suit the expand with per shard parrallelization, + // because the property_name_map has to be accessed from multiple threads + // in order to avoid duplicated keys (two threads might map the same + // property with different numbers) and multiple passes (to unify the + // mapping amond result returned from different shards). + 1: list<ExpandOneResultRow> result; + 2: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map; +} + +struct NewVertex { + 1: list<i64> label_ids; + 2: map<i64, Value> properties; +} + +struct CreateVerticesRequest { + 1: required i64 transaction_id; + 2: map<i64, binary> (cpp.template = "std::unordered_map") labels_name_map; + 3: map<i64, binary> (cpp.template = "std::unordered_map") property_name_map; + 4: list<NewVertex> new_vertices; +} + + +service Storage { + i64 startTransaction() + Result commitTransaction(1: i64 transaction_id) + void abortTransaction(1: i64 transaction_id) + + Result createVertices(1: CreateVerticesRequest req) + ScanVerticesResponse scanVertices(1: ScanVerticesRequest req) + GetPropertiesResponse getProperties(1: GetPropertiesRequest req) + ExpandOneResponse expandOne(1: ExpandOneRequest req) + +} diff --git a/src/interface/trial.thrift b/src/interface/trial.thrift new file mode 100644 index 000000000..6c89caef3 --- /dev/null +++ b/src/interface/trial.thrift @@ -0,0 +1,20 @@ +struct Ping { + 1: binary message; +} + +struct Pong{ + 1: binary message; +} + +struct ValueToAdd{ + 1: i64 val; +} + +struct ValueToAddResopnse{ + 1: i64 new_val; +} + +service PingPong { + Pong ping(1: Ping req) + ValueToAddResopnse AddValue(1: ValueToAdd req) +} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c4c957b8b..db93aef02 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -1,8 +1,11 @@ +# include(MgThrift) + set(test_prefix memgraph__unit__) find_package(fmt REQUIRED) find_package(gflags REQUIRED) find_package(Threads REQUIRED) +find_package(Folly REQUIRED) add_custom_target(memgraph__unit) @@ -37,7 +40,7 @@ function(_add_unit_test test_cpp custom_main) # used to help create two targets of the same name even though CMake # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) - target_link_libraries(${target_name} mg-memory mg-utils gtest gmock Threads::Threads dl) + target_link_libraries(${target_name} mg-memory mg-utils gtest gmock Threads::Threads dl mg-interface-echo-cpp2) # register test if(TEST_COVERAGE) @@ -397,4 +400,4 @@ target_link_libraries(${test_prefix}future mg-io) # Test Thrift transport echo add_unit_test(thrift_transport_echo.cpp) -target_link_libraries(${test_prefix}thrift_transport_echo mg-io) +target_link_libraries(${test_prefix}thrift_transport_echo mg-io Threads::Threads mg-interface-echo-cpp2 fmt) diff --git a/tests/unit/main.cpp b/tests/unit/main.cpp index 88d266b24..a81ed608c 100644 --- a/tests/unit/main.cpp +++ b/tests/unit/main.cpp @@ -9,11 +9,15 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include <folly/init/Init.h> #include <gtest/gtest.h> #include <utils/logging.hpp> int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); + + folly::Init(&argc, &argv); + memgraph::logging::RedirectToStderr(); spdlog::set_level(spdlog::level::trace); return RUN_ALL_TESTS(); diff --git a/tests/unit/thrift_transport_echo.cpp b/tests/unit/thrift_transport_echo.cpp index af8cd49c2..679011724 100644 --- a/tests/unit/thrift_transport_echo.cpp +++ b/tests/unit/thrift_transport_echo.cpp @@ -14,11 +14,111 @@ #include "gtest/gtest.h" +#include <folly/init/Init.h> +#include <folly/io/SocketOptionMap.h> +#include <folly/io/async/AsyncServerSocket.h> +#include <folly/net/NetworkSocket.h> +#include <thrift/lib/cpp2/async/HeaderClientChannel.h> +#include <thrift/lib/cpp2/server/ThriftServer.h> +#include "interface/gen-cpp2/Echo.h" // From generated code +#include "interface/gen-cpp2/EchoAsyncClient.h" + #include "io/thrift/thrift_transport.hpp" using namespace memgraph::io; +using namespace apache::thrift; +using namespace cpp2; +using namespace folly; + +using namespace std::chrono_literals; + +// static constexpr int port = 6666; // The port on which server is listening + +class EchoSvc : public EchoSvIf { + inline static const std::string prefix_{"0"}; + + std::string current_message_; + // bool has_message_{false}; + + public: + virtual ~EchoSvc() {} + + // The Thrift handle method + void ReceiveSend(const EchoMessage &m) override { + // m.get_message(); + // LOG(ERROR) << "Received\n"; + current_message_ = prefix_ + m.get_message(); + // SendOutMessage(6665); + // LOG(ERROR) << "Sent\n"; + } + + void SendOneShotMessage(int other_port, const std::string &message_str) { + EventBase base; + auto socket(folly::AsyncSocket::newSocket(&base, "127.0.0.1", other_port)); + + // Create a HeaderClientChannel object which is used in creating + // client object + auto client_channel = HeaderClientChannel::newChannel(std::move(socket)); + // Create a client object + EchoAsyncClient client(std::move(client_channel)); + + EchoMessage message; + message.message_ref() = message_str; + client.sync_ReceiveSend(message); + } + + void SendOutMessage(int other_port) { + EventBase base; + auto socket(folly::AsyncSocket::newSocket(&base, "127.0.0.1", other_port)); + + // Create a HeaderClientChannel object which is used in creating + // client object + auto client_channel = HeaderClientChannel::newChannel(std::move(socket)); + // Create a client object + EchoAsyncClient client(std::move(client_channel)); + + EchoMessage message; + message.message_ref() = current_message_; + client.sync_ReceiveSend(message); + } + + std::string GetCurrentMessage() { return current_message_; } +}; + TEST(ThriftTransport, Echo) { // TODO(tyler and gabor) use thrift-generated echo, and thrift transport, to send, reply, and receive the response for // a thrift-defined message + auto ptr1 = std::make_shared<EchoSvc>(); + auto ptr2 = std::make_shared<EchoSvc>(); + + auto server_thread2 = std::jthread([&ptr2] { + ThriftServer *s = new ThriftServer(); + s->setInterface(ptr2); + s->setPort(6666); + s->serve(); + }); + + auto server_thread1 = std::jthread([&ptr1] { + ThriftServer *s = new ThriftServer(); + s->setInterface(ptr1); + s->setPort(6665); + s->serve(); + }); + + std::this_thread::sleep_for(4000ms); + + ptr1->SendOneShotMessage(6666, "original"); + std::this_thread::sleep_for(4000ms); + + ptr2->SendOutMessage(6665); + std::this_thread::sleep_for(4000ms); + + auto result = ptr1->GetCurrentMessage(); + + ASSERT_EQ(result, std::string("00original")); + + // Solve this once this is not just a POC. + server_thread2.detach(); + server_thread1.detach(); }