From e8e4cd7f977f9ab459bfb3da03d12a2750f3ffe8 Mon Sep 17 00:00:00 2001 From: antonio2368 Date: Fri, 19 Mar 2021 08:43:48 +0100 Subject: [PATCH] Memory control e2e test (#115) * Add memory control e2e test * Fix cmake for jemalloc --- .github/workflows/diff.yaml | 8 ++++ .github/workflows/release_centos8.yaml | 8 ++++ .github/workflows/release_debian10.yaml | 8 ++++ .github/workflows/release_ubuntu2004.yaml | 8 ++++ libs/jemalloc.cmake | 2 +- tests/e2e/CMakeLists.txt | 2 + tests/e2e/memory/CMakeLists.txt | 2 + tests/e2e/memory/memory_control.cpp | 56 +++++++++++++++++++++++ tests/e2e/memory/workloads.yaml | 15 ++++++ tests/e2e/runner.py | 2 +- 10 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/memory/CMakeLists.txt create mode 100644 tests/e2e/memory/memory_control.cpp create mode 100644 tests/e2e/memory/workloads.yaml diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index 10bf2de97..50ede0351 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -299,6 +299,14 @@ jobs: cd e2e LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml + - name: Run e2e memory control tests + run: | + cd tests + ./setup.sh + source ve3/bin/activate + cd e2e + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml + - name: Run stress test (plain) run: | cd tests/stress diff --git a/.github/workflows/release_centos8.yaml b/.github/workflows/release_centos8.yaml index 49584f5bd..b7433416d 100644 --- a/.github/workflows/release_centos8.yaml +++ b/.github/workflows/release_centos8.yaml @@ -292,6 +292,14 @@ jobs: cd e2e LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml + - name: Run e2e memory control tests + run: | + cd tests + ./setup.sh + source ve3/bin/activate + cd e2e + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml + - name: Run stress test (plain) run: | cd tests/stress diff --git a/.github/workflows/release_debian10.yaml b/.github/workflows/release_debian10.yaml index 7a728fef8..da94818ea 100644 --- a/.github/workflows/release_debian10.yaml +++ b/.github/workflows/release_debian10.yaml @@ -290,6 +290,14 @@ jobs: cd e2e LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml + - name: Run e2e memory control tests + run: | + cd tests + ./setup.sh + source ve3/bin/activate + cd e2e + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml + - name: Run stress test (plain) run: | cd tests/stress diff --git a/.github/workflows/release_ubuntu2004.yaml b/.github/workflows/release_ubuntu2004.yaml index 90386fce5..41421b5a7 100644 --- a/.github/workflows/release_ubuntu2004.yaml +++ b/.github/workflows/release_ubuntu2004.yaml @@ -290,6 +290,14 @@ jobs: cd e2e LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml + - name: Run e2e memory control tests + run: | + cd tests + ./setup.sh + source ve3/bin/activate + cd e2e + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml + - name: Run stress test (plain) run: | cd tests/stress diff --git a/libs/jemalloc.cmake b/libs/jemalloc.cmake index 4625f98bc..390b92a44 100644 --- a/libs/jemalloc.cmake +++ b/libs/jemalloc.cmake @@ -47,6 +47,6 @@ endif() target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) # for RTLD_NEXT -target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) +target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE) set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 29749fd19..9d6d7bd8e 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -1 +1,3 @@ add_subdirectory(replication) + +add_subdirectory(memory) diff --git a/tests/e2e/memory/CMakeLists.txt b/tests/e2e/memory/CMakeLists.txt new file mode 100644 index 000000000..4e258f61a --- /dev/null +++ b/tests/e2e/memory/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(memgraph__e2e__memory__control memory_control.cpp) +target_link_libraries(memgraph__e2e__memory__control gflags mgclient mg-utils mg-io Threads::Threads) diff --git a/tests/e2e/memory/memory_control.cpp b/tests/e2e/memory/memory_control.cpp new file mode 100644 index 000000000..537fba5e7 --- /dev/null +++ b/tests/e2e/memory/memory_control.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include "utils/logging.hpp" +#include "utils/timer.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); +DEFINE_uint64(timeout, 120, "Timeout seconds"); + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Memory Control"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + logging::RedirectToStderr(); + + mg::Client::Init(); + + auto client = + mg::Client::Connect({.host = "127.0.0.1", .port = static_cast(FLAGS_bolt_port), .use_ssl = false}); + if (!client) { + LOG_FATAL("Failed to connect!"); + } + + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + + const auto *create_query = "UNWIND range(1, 50) as u CREATE (n {string: \"Some longer string\"}) RETURN n;"; + + utils::Timer timer; + while (true) { + if (timer.Elapsed>().count() > FLAGS_timeout) { + LOG_FATAL("The test timed out"); + } + client->Execute(create_query); + if (!client->FetchOne()) { + break; + } + client->DiscardAll(); + } + + spdlog::info("Memgraph is out of memory"); + + spdlog::info("Cleaning up unused memory"); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + client->Execute("FREE MEMORY;"); + client->DiscardAll(); + + // now it should succeed + spdlog::info("Retrying the query with the memory cleaned up"); + client->Execute(create_query); + if (!client->FetchOne()) { + LOG_FATAL("Memgraph is still out of memory"); + } + + return 0; +} diff --git a/tests/e2e/memory/workloads.yaml b/tests/e2e/memory/workloads.yaml new file mode 100644 index 000000000..147b87b04 --- /dev/null +++ b/tests/e2e/memory/workloads.yaml @@ -0,0 +1,15 @@ +bolt_port: &bolt_port "7687" +template_cluster: &template_cluster + cluster: + main: + args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"] + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Memory control" + binary: "tests/e2e/memory/memgraph__e2e__memory__control" + args: ["--bolt-port", *bolt_port, "--timeout", "180"] + <<: *template_cluster + + diff --git a/tests/e2e/runner.py b/tests/e2e/runner.py index d1d29ac23..453c8d3de 100755 --- a/tests/e2e/runner.py +++ b/tests/e2e/runner.py @@ -12,7 +12,7 @@ PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..")) BUILD_DIR = os.path.join(PROJECT_DIR, "build") MEMGRAPH_BINARY = os.path.join(BUILD_DIR, "memgraph") -log = logging.getLogger("memgraph.tests.e2e.replication") +log = logging.getLogger("memgraph.tests.e2e") def load_args():