From 3e7aef432f3d9745d2b268d2a6569cb34a5a936f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Tue, 10 May 2022 13:31:45 +0200 Subject: [PATCH 1/6] Improve docker-compose setup for running streams e2e tests locally (#388) * Separate pulsar and kafka docker-compose files * Add docker-compose for redpanda * Add small docs how to use docker-compose files --- tests/e2e/streams/README.md | 8 +++++++ .../streams/{docker-compose.yml => kafka.yml} | 12 +++------- tests/e2e/streams/pulsar.yml | 8 +++++++ tests/e2e/streams/redpanda.yml | 23 +++++++++++++++++++ 4 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/streams/README.md rename tests/e2e/streams/{docker-compose.yml => kafka.yml} (62%) create mode 100644 tests/e2e/streams/pulsar.yml create mode 100644 tests/e2e/streams/redpanda.yml diff --git a/tests/e2e/streams/README.md b/tests/e2e/streams/README.md new file mode 100644 index 000000000..1412f56b2 --- /dev/null +++ b/tests/e2e/streams/README.md @@ -0,0 +1,8 @@ +There are three docker-compose files in this directory: +* [kafka.yml](kafka.yml) +* [pulsar.yml](pulsar.yml) +* [redpanda.yml](redpanda.yml) + +To run one of them, use the `docker-compose -f up -V` command. Optionally you can append `-d` to detach from the started containers. You can stop the detach containers by `docker-compose -f down`. + +If you experience strange errors, try to clean up the previously created containers by `docker-compose -f rm -svf`. \ No newline at end of file diff --git a/tests/e2e/streams/docker-compose.yml b/tests/e2e/streams/kafka.yml similarity index 62% rename from tests/e2e/streams/docker-compose.yml rename to tests/e2e/streams/kafka.yml index 3d0778751..f9d213864 100644 --- a/tests/e2e/streams/docker-compose.yml +++ b/tests/e2e/streams/kafka.yml @@ -1,13 +1,13 @@ -version: "3" +version: '3.7' services: zookeeper: - image: 'bitnami/zookeeper:3.6.3-debian-10-r33' + image: 'bitnami/zookeeper:latest' ports: - '2181:2181' environment: - ALLOW_ANONYMOUS_LOGIN=yes kafka: - image: 'bitnami/kafka:2.8.0-debian-10-r49' + image: 'bitnami/kafka:latest' ports: - '9092:9092' environment: @@ -18,9 +18,3 @@ services: - ALLOW_PLAINTEXT_LISTENER=yes depends_on: - zookeeper - pulsar: - image: 'apachepulsar/pulsar:2.8.1' - ports: - - '6652:8080' - - '6650:6650' - entrypoint: ['bin/pulsar', 'standalone'] diff --git a/tests/e2e/streams/pulsar.yml b/tests/e2e/streams/pulsar.yml new file mode 100644 index 000000000..31241ad66 --- /dev/null +++ b/tests/e2e/streams/pulsar.yml @@ -0,0 +1,8 @@ +version: '3.7' +services: + pulsar: + image: 'apachepulsar/pulsar:latest' + ports: + - '6652:8080' + - '6650:6650' + entrypoint: ['bin/pulsar', 'standalone'] diff --git a/tests/e2e/streams/redpanda.yml b/tests/e2e/streams/redpanda.yml new file mode 100644 index 000000000..3b7603b16 --- /dev/null +++ b/tests/e2e/streams/redpanda.yml @@ -0,0 +1,23 @@ +version: '3.7' +services: + redpanda: + command: + - redpanda + - start + - --smp + - '1' + - --reserve-memory + - 0M + - --overprovisioned + - --node-id + - '0' + - --kafka-addr + - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr + - PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 + # NOTE: Please use the latest version here! + image: docker.vectorized.io/vectorized/redpanda:latest + container_name: redpanda-1 + ports: + - 9092:9092 + - 29092:29092 From 483f4d04bd12fa1d01c2159d9ab7f283ebb9790f Mon Sep 17 00:00:00 2001 From: Jure Bajic Date: Wed, 11 May 2022 21:07:04 +0200 Subject: [PATCH 2/6] Update precommit hooks (#393) * Add pre-commit * Add clang pre-commit * Install pre-commit in init --- .githooks/pre-commit | 9 --------- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ init | 4 ++++ 3 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 13612b158..df8c02d42 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -24,14 +24,6 @@ for file in $modified_files; do git checkout-index --prefix="$tmpdir/" -- $file - echo "Running clang-format..." - $project_folder/tools/git-clang-format $tmpdir/$file - CODE=$? - - if [ $CODE -ne 0 ]; then - break - fi - # Do not break header checker echo "Running header checker..." $project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year @@ -39,7 +31,6 @@ for file in $modified_files; do if [ $CODE -ne 0 ]; then FAIL=1 fi - done; return ${FAIL} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..869991668 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + args: # arguments to configure black + - --line-length=120 + - --include='\.pyi?$' + # these folders wont be formatted by black + - --exclude="""\.git | + \.__pycache__| + build| + libs| + .cache""" + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v13.0.0 + hooks: + - id: clang-format diff --git a/init b/init index b0fa6e4ea..b74f019bb 100755 --- a/init +++ b/init @@ -135,3 +135,7 @@ for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook" echo "Added $hook hook" done; + +# Install precommit hook +python3 -m pip install pre-commit +python3 -m pre_commit install From a7f4c98bea0bb00b2670f83b6297b3f1ca5153ec Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 13 May 2022 11:43:10 +0200 Subject: [PATCH 3/6] Fix module symbol loading (#335) * Use DEEPBIND * Add dependency on libstdc++ Co-authored-by: Antonio Andelic Co-authored-by: Jure Bajic --- release/CMakeLists.txt | 4 ++-- src/memgraph.cpp | 16 ++++++++++++++++ src/query/procedure/module.cpp | 3 ++- src/query/procedure/module.hpp | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/release/CMakeLists.txt b/release/CMakeLists.txt index f77d49dc7..0fb250db8 100644 --- a/release/CMakeLists.txt +++ b/release/CMakeLists.txt @@ -41,7 +41,7 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY} applications driver by real-time connected data.") # Add `openssl` package to dependencies list. Used to generate SSL certificates. # We also depend on `python3` because we embed it in Memgraph. -set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6") # Setting arhitecture extension for rpm packages set(MG_ARCH_EXTENSION_RPM "noarch") @@ -67,7 +67,7 @@ It aims to deliver developers the speed, simplicity and scale required to build the next generation of applications driver by real-time connected data.") # Add `openssl` package to dependencies list. Used to generate SSL certificates. # We also depend on `python3` because we embed it in Memgraph. -set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0") +set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6") # All variables must be set before including. include(CPack) diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 0554eca8b..f0872f647 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -1070,6 +1070,22 @@ int main(int argc, char **argv) { if (maybe_exc) { spdlog::error(memgraph::utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc, "https://memgr.ph/python")); + } else { + // Change how we load dynamic libraries on Python by using RTLD_NOW and + // RTLD_DEEPBIND flags. This solves an issue with using the wrong version of + // libstd. + auto gil = memgraph::py::EnsureGIL(); + // NOLINTNEXTLINE(hicpp-signed-bitwise) + auto *flag = PyLong_FromLong(RTLD_NOW | RTLD_DEEPBIND); + auto *setdl = PySys_GetObject("setdlopenflags"); + MG_ASSERT(setdl); + auto *arg = PyTuple_New(1); + MG_ASSERT(arg); + MG_ASSERT(PyTuple_SetItem(arg, 0, flag) == 0); + PyObject_CallObject(setdl, arg); + Py_DECREF(flag); + Py_DECREF(setdl); + Py_DECREF(arg); } } else { spdlog::error( diff --git a/src/query/procedure/module.cpp b/src/query/procedure/module.cpp index af3837048..5c912aaf0 100644 --- a/src/query/procedure/module.cpp +++ b/src/query/procedure/module.cpp @@ -822,7 +822,8 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) { spdlog::info("Loading module {}...", file_path); file_path_ = file_path; dlerror(); // Clear any existing error. - handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL); + // NOLINTNEXTLINE(hicpp-signed-bitwise) + handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND); if (!handle_) { spdlog::error( utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules")); diff --git a/src/query/procedure/module.hpp b/src/query/procedure/module.hpp index c3403b636..17a62d738 100644 --- a/src/query/procedure/module.hpp +++ b/src/query/procedure/module.hpp @@ -13,6 +13,7 @@ /// API for loading and registering modules providing custom oC procedures #pragma once +#include #include #include #include @@ -128,6 +129,40 @@ class ModuleRegistry final { const std::filesystem::path &InternalModuleDir() const noexcept; private: + class SharedLibraryHandle { + public: + SharedLibraryHandle(const std::string &shared_library, int mode) : handle_{dlopen(shared_library.c_str(), mode)} {} + SharedLibraryHandle(const SharedLibraryHandle &) = delete; + SharedLibraryHandle(SharedLibraryHandle &&) = delete; + SharedLibraryHandle operator=(const SharedLibraryHandle &) = delete; + SharedLibraryHandle operator=(SharedLibraryHandle &&) = delete; + + ~SharedLibraryHandle() { + if (handle_) { + dlclose(handle_); + } + } + + private: + void *handle_; + }; + +#if __has_feature(address_sanitizer) + // This is why we need RTLD_NODELETE and we must not use RTLD_DEEPBIND with + // ASAN: https://github.com/google/sanitizers/issues/89 + SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE}; +#else + // The reason behind opening share library during runtime is to avoid issues + // with loading symbols from stdlib. We have encounter issues with locale + // that cause std::cout not being printed and issues when python libraries + // would call stdlib (e.g. pytorch). + // The way that those issues were solved was + // by using RTLD_DEEPBIND. RTLD_DEEPBIND ensures that the lookup for the + // mentioned library will be first performed in the already existing binded + // libraries and then the global namespace. + // RTLD_DEEPBIND => https://linux.die.net/man/3/dlopen + SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND}; +#endif std::vector modules_dirs_; std::filesystem::path internal_module_dir_; }; From 8059a3e6533a2033ba35c9c1eca547506b6c7012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Tue, 17 May 2022 15:33:28 +0200 Subject: [PATCH 4/6] Improve magic functions docs (#391) * Improve documentation of magic functions * Improve wording for decorators --- include/mg_procedure.h | 2 +- include/mgp.py | 71 ++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/include/mg_procedure.h b/include/mg_procedure.h index 5d2527e81..b4a9f7592 100644 --- a/include/mg_procedure.h +++ b/include/mg_procedure.h @@ -1559,7 +1559,7 @@ enum mgp_error mgp_func_add_opt_arg(struct mgp_func *func, const char *name, str typedef void (*mgp_func_cb)(struct mgp_list *, struct mgp_func_context *, struct mgp_func_result *, struct mgp_memory *); -/// Register a Memgraph magic function +/// Register a Memgraph magic function. /// /// The `name` must be a sequence of digits, underscores, lowercase and /// uppercase Latin letters. The name must begin with a non-digit character. diff --git a/include/mgp.py b/include/mgp.py index 7b9d2dcb9..73b71d610 100644 --- a/include/mgp.py +++ b/include/mgp.py @@ -1179,16 +1179,15 @@ def read_proc(func: typing.Callable[..., Record]): """ Register `func` as a read-only procedure of the current module. - `read_proc` is meant to be used as a decorator function to register module - procedures. The registered `func` needs to be a callable which optionally - takes `ProcCtx` as the first argument. Other arguments of `func` will be - bound to values passed in the cypherQuery. The full signature of `func` - needs to be annotated with types. The return type must be - `Record(field_name=type, ...)` and the procedure must produce either a - complete Record or None. To mark a field as deprecated, use - `Record(field_name=Deprecated(type), ...)`. Multiple records can be - produced by returning an iterable of them. Registering generator functions - is currently not supported. + The decorator `read_proc` is meant to be used to register module procedures. + The registered `func` needs to be a callable which optionally takes + `ProcCtx` as its first argument. Other arguments of `func` will be bound to + values passed in the cypherQuery. The full signature of `func` needs to be + annotated with types. The return type must be `Record(field_name=type, ...)` + and the procedure must produce either a complete Record or None. To mark a + field as deprecated, use `Record(field_name=Deprecated(type), ...)`. + Multiple records can be produced by returning an iterable of them. + Registering generator functions is currently not supported. Example usage. @@ -1222,16 +1221,16 @@ def write_proc(func: typing.Callable[..., Record]): """ Register `func` as a writeable procedure of the current module. - `write_proc` is meant to be used as a decorator function to register module + The decorator `write_proc` is meant to be used to register module procedures. The registered `func` needs to be a callable which optionally takes `ProcCtx` as the first argument. Other arguments of `func` will be bound to values passed in the cypherQuery. The full signature of `func` needs to be annotated with types. The return type must be `Record(field_name=type, ...)` and the procedure must produce either a complete Record or None. To mark a field as deprecated, use - `Record(field_name=Deprecated(type), ...)`. Multiple records can be - produced by returning an iterable of them. Registering generator functions - is currently not supported. + `Record(field_name=Deprecated(type), ...)`. Multiple records can be produced + by returning an iterable of them. Registering generator functions is + currently not supported. Example usage. @@ -1459,8 +1458,9 @@ def transformation(func: typing.Callable[..., Record]): class FuncCtx: """Context of a function being executed. - Access to a FuncCtx is only valid during a single execution of a transformation. - You should not globally store a FuncCtx instance. + Access to a FuncCtx is only valid during a single execution of a function in + a query. You should not globally store a FuncCtx instance. The graph object + within the FuncCtx is not mutable. """ __slots__ = "_graph" @@ -1475,6 +1475,45 @@ class FuncCtx: def function(func: typing.Callable): + """ + Register `func` as a user-defined function in the current module. + + The decorator `function` is meant to be used to register module functions. + The registered `func` needs to be a callable which optionally takes + `FuncCtx` as its first argument. Other arguments of `func` will be bound to + values passed in the Cypher query. Only the funcion arguments need to be + annotated with types. The return type doesn't need to be specified, but it + has to be supported by `mgp.Any`. Registering generator functions is + currently not supported. + + Example usage. + + ``` + import mgp + @mgp.function + def func_example(context: mgp.FuncCtx, + required_arg: str, + optional_arg: mgp.Nullable[str] = None + ): + return_args = [required_arg] + if optional_arg is not None: + return_args.append(optional_arg) + # Return any kind of result supported by mgp.Any + return return_args + ``` + + The example function above returns a list of provided arguments: + * `required_arg` is always present and its value is the first argument of + the function. + * `optional_arg` is present if the second argument of the function is not + `null`. + Any errors can be reported by raising an Exception. + + The function can be invoked in Cypher using the following calls: + RETURN example.func_example("first argument", "second_argument"); + RETURN example.func_example("first argument"); + Naturally, you may pass in different arguments. + """ raise_if_does_not_meet_requirements(func) register_func = _mgp.Module.add_function sig = inspect.signature(func) From 22bd60c613415a06997441932c21819646230bca Mon Sep 17 00:00:00 2001 From: Jure Bajic Date: Wed, 18 May 2022 07:50:06 +0200 Subject: [PATCH 5/6] Fix shutdown call (#395) * Fix shutdown not called * Add ssl server tests --- src/communication/v2/session.hpp | 7 +-- tests/e2e/CMakeLists.txt | 5 +- .../{websocket => }/memgraph-selfsigned.crt | 0 .../{websocket => }/memgraph-selfsigned.key | 0 tests/e2e/monitoring_server/CMakeLists.txt | 8 +++ .../common.hpp | 0 .../monitoring.cpp} | 0 .../monitoring_ssl.cpp} | 0 .../workloads.yaml | 21 ++++--- tests/e2e/server/CMakeLists.txt | 8 +++ tests/e2e/server/common.hpp | 60 +++++++++++++++++++ tests/e2e/server/server_connection.cpp | 56 +++++++++++++++++ tests/e2e/server/server_ssl_connection.cpp | 57 ++++++++++++++++++ tests/e2e/server/workloads.yaml | 34 +++++++++++ tests/e2e/websocket/CMakeLists.txt | 10 ---- 15 files changed, 238 insertions(+), 28 deletions(-) rename tests/e2e/{websocket => }/memgraph-selfsigned.crt (100%) rename tests/e2e/{websocket => }/memgraph-selfsigned.key (100%) create mode 100644 tests/e2e/monitoring_server/CMakeLists.txt rename tests/e2e/{websocket => monitoring_server}/common.hpp (100%) rename tests/e2e/{websocket/websocket.cpp => monitoring_server/monitoring.cpp} (100%) rename tests/e2e/{websocket/websocket_ssl.cpp => monitoring_server/monitoring_ssl.cpp} (100%) rename tests/e2e/{websocket => monitoring_server}/workloads.yaml (59%) create mode 100644 tests/e2e/server/CMakeLists.txt create mode 100644 tests/e2e/server/common.hpp create mode 100644 tests/e2e/server/server_connection.cpp create mode 100644 tests/e2e/server/server_ssl_connection.cpp create mode 100644 tests/e2e/server/workloads.yaml delete mode 100644 tests/e2e/websocket/CMakeLists.txt diff --git a/src/communication/v2/session.hpp b/src/communication/v2/session.hpp index 0add8244d..62bc4346f 100644 --- a/src/communication/v2/session.hpp +++ b/src/communication/v2/session.hpp @@ -246,11 +246,7 @@ class Session final : public std::enable_shared_from_this +#include +#include + +#include +#include +#include +#include +#include + +#include "utils/logging.hpp" + +inline void OnTimeoutExpiration(const boost::system::error_code &ec) { + // Timer was not cancelled, take necessary action. + MG_ASSERT(!!ec, "Connection timeout"); +} + +inline void EstablishConnection(const uint16_t bolt_port, const bool use_ssl) { + spdlog::info("Testing successfull connection from one client"); + mg::Client::Init(); + + boost::asio::io_context ioc; + boost::asio::steady_timer timer(ioc, std::chrono::seconds(5)); + timer.async_wait(std::bind_front(&OnTimeoutExpiration)); + std::jthread bg_thread([&ioc]() { ioc.run(); }); + + auto client = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = use_ssl}); + MG_ASSERT(client, "Failed to connect!"); + timer.cancel(); +} + +inline void EstablishMultipleConnections(const uint16_t bolt_port, const bool use_ssl) { + spdlog::info("Testing successfull connection from multiple clients"); + mg::Client::Init(); + + boost::asio::io_context ioc; + boost::asio::steady_timer timer(ioc, std::chrono::seconds(5)); + timer.async_wait(std::bind_front(&OnTimeoutExpiration)); + std::jthread bg_thread([&ioc]() { ioc.run(); }); + + auto client1 = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = use_ssl}); + auto client2 = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = use_ssl}); + auto client3 = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = use_ssl}); + + MG_ASSERT(client1, "Failed to connect!"); + MG_ASSERT(client2, "Failed to connect!"); + MG_ASSERT(client3, "Failed to connect!"); + timer.cancel(); +} diff --git a/tests/e2e/server/server_connection.cpp b/tests/e2e/server/server_connection.cpp new file mode 100644 index 000000000..7f3f31c4d --- /dev/null +++ b/tests/e2e/server/server_connection.cpp @@ -0,0 +1,56 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common.hpp" +#include "utils/logging.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); + +void EstablishSSLConnectionToNonSSLServer(const auto bolt_port) { + spdlog::info("Testing that connection fails when connecting to non SSL server while using SSL"); + mg::Client::Init(); + + boost::asio::io_context ioc; + boost::asio::steady_timer timer(ioc, std::chrono::seconds(5)); + timer.async_wait(std::bind_front(&OnTimeoutExpiration)); + std::jthread bg_thread([&ioc]() { ioc.run(); }); + + auto client = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = true}); + + MG_ASSERT(client == nullptr, "Connection not refused when connecting with SSL turned on to a non SSL server!"); + timer.cancel(); +} + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E server connection!"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + MG_ASSERT(FLAGS_bolt_port != 0); + memgraph::logging::RedirectToStderr(); + + const auto bolt_port = static_cast(FLAGS_bolt_port); + + EstablishConnection(bolt_port, false); + EstablishMultipleConnections(bolt_port, false); + EstablishSSLConnectionToNonSSLServer(bolt_port); + + return 0; +} diff --git a/tests/e2e/server/server_ssl_connection.cpp b/tests/e2e/server/server_ssl_connection.cpp new file mode 100644 index 000000000..5368b5e65 --- /dev/null +++ b/tests/e2e/server/server_ssl_connection.cpp @@ -0,0 +1,57 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common.hpp" +#include "utils/logging.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); + +void EstablishNonSSLConnectionToSSLServer(const auto bolt_port) { + spdlog::info("Testing that connection fails when connecting to SSL server without using SSL"); + mg::Client::Init(); + + boost::asio::io_context ioc; + boost::asio::steady_timer timer(ioc, std::chrono::seconds(5)); + timer.async_wait(std::bind_front(&OnTimeoutExpiration)); + std::jthread bg_thread([&ioc]() { ioc.run(); }); + + auto client = mg::Client::Connect({.host = "127.0.0.1", .port = bolt_port, .use_ssl = false}); + + MG_ASSERT(client == nullptr, "Connection not refused when conneting without SSL turned on to a SSL server!"); + timer.cancel(); +} + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E server SSL connection!"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + MG_ASSERT(FLAGS_bolt_port != 0); + memgraph::logging::RedirectToStderr(); + + const auto bolt_port = static_cast(FLAGS_bolt_port); + + EstablishConnection(bolt_port, true); + EstablishMultipleConnections(bolt_port, true); + EstablishNonSSLConnectionToSSLServer(bolt_port); + + return 0; +} diff --git a/tests/e2e/server/workloads.yaml b/tests/e2e/server/workloads.yaml new file mode 100644 index 000000000..db956e701 --- /dev/null +++ b/tests/e2e/server/workloads.yaml @@ -0,0 +1,34 @@ +cert_file: &cert_file "$PROJECT_DIR/tests/e2e/memgraph-selfsigned.crt" +key_file: &key_file "$PROJECT_DIR/tests/e2e/memgraph-selfsigned.key" +bolt_port: &bolt_port "7687" +template_cluster: &template_cluster + cluster: + server: + args: ["--bolt-port=7687", "--log-level=TRACE", "--"] + log_file: "server-connection-e2e.log" +template_cluster_ssl: &template_cluster_ssl + cluster: + server: + args: + [ + "--bolt-port", + *bolt_port, + "--log-level=TRACE", + "--bolt-cert-file", + *cert_file, + "--bolt-key-file", + *key_file, + "--", + ] + log_file: "server-connection-ssl-e2e.log" + ssl: true + +workloads: + - name: "Server connection" + binary: "tests/e2e/server/memgraph__e2e__server_connection" + args: ["--bolt-port", *bolt_port] + <<: *template_cluster + - name: "Server SSL connection" + binary: "tests/e2e/server/memgraph__e2e__server_ssl_connection" + args: ["--bolt-port", *bolt_port] + <<: *template_cluster_ssl diff --git a/tests/e2e/websocket/CMakeLists.txt b/tests/e2e/websocket/CMakeLists.txt deleted file mode 100644 index 15c3a0f3a..000000000 --- a/tests/e2e/websocket/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -find_package(gflags REQUIRED) -find_package(Boost REQUIRED) - -add_executable(memgraph__e2e__websocket websocket.cpp) -target_link_libraries(memgraph__e2e__websocket mgclient mg-utils json gflags Boost::headers) - -add_executable(memgraph__e2e__websocket_ssl websocket_ssl.cpp) -target_link_libraries(memgraph__e2e__websocket_ssl mgclient mg-utils json gflags Boost::headers) -file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/memgraph-selfsigned.crt DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/memgraph-selfsigned.key DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) From cccf32e79d88d93e46704294c6936dfdbff3f563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Tue, 17 May 2022 23:34:07 -0700 Subject: [PATCH 6/6] Add default commented values for Bolt SSL flags (#398) --- config/flags.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/flags.yaml b/config/flags.yaml index c3e5e754c..6dd50e50c 100644 --- a/config/flags.yaml +++ b/config/flags.yaml @@ -47,6 +47,14 @@ modifications: value: "" override: false + - name: "bolt_cert_file" + value: "/etc/memgraph/ssl/cert.pem" + override: false + + - name: "bolt_key_file" + value: "/etc/memgraph/ssl/key.pem" + override: false + - name: "storage_properties_on_edges" value: "true" override: true