Memory control e2e test (#115)

* Add memory control e2e test

* Fix cmake for jemalloc

.github/workflows/diff.yaml

@@ -299,6 +299,14 @@ jobs:
           cd e2e
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
 
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

.github/workflows/release_centos8.yaml

@@ -292,6 +292,14 @@ jobs:
           cd e2e
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
 
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

.github/workflows/release_debian10.yaml

@@ -290,6 +290,14 @@ jobs:
           cd e2e
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
 
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

.github/workflows/release_ubuntu2004.yaml

@@ -290,6 +290,14 @@ jobs:
           cd e2e
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
 
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

libs/jemalloc.cmake

@@ -47,6 +47,6 @@ endif()
 
 target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
 # for RTLD_NEXT
-target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE)
+target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
 
 set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

tests/e2e/CMakeLists.txt

@@ -1 +1,3 @@
 add_subdirectory(replication)
+
+add_subdirectory(memory)

tests/e2e/memory/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable(memgraph__e2e__memory__control memory_control.cpp)
+target_link_libraries(memgraph__e2e__memory__control gflags mgclient mg-utils mg-io Threads::Threads)

tests/e2e/memory/memory_control.cpp

@@ -0,0 +1,56 @@
+#include <gflags/gflags.h>
+#include <mgclient.hpp>
+
+#include "utils/logging.hpp"
+#include "utils/timer.hpp"
+
+DEFINE_uint64(bolt_port, 7687, "Bolt port");
+DEFINE_uint64(timeout, 120, "Timeout seconds");
+
+int main(int argc, char **argv) {
+  google::SetUsageMessage("Memgraph E2E Memory Control");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  logging::RedirectToStderr();
+
+  mg::Client::Init();
+
+  auto client =
+      mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
+  if (!client) {
+    LOG_FATAL("Failed to connect!");
+  }
+
+  client->Execute("MATCH (n) DETACH DELETE n;");
+  client->DiscardAll();
+
+  const auto *create_query = "UNWIND range(1, 50) as u CREATE (n {string: \"Some longer string\"}) RETURN n;";
+
+  utils::Timer timer;
+  while (true) {
+    if (timer.Elapsed<std::chrono::duration<uint64_t>>().count() > FLAGS_timeout) {
+      LOG_FATAL("The test timed out");
+    }
+    client->Execute(create_query);
+    if (!client->FetchOne()) {
+      break;
+    }
+    client->DiscardAll();
+  }
+
+  spdlog::info("Memgraph is out of memory");
+
+  spdlog::info("Cleaning up unused memory");
+  client->Execute("MATCH (n) DETACH DELETE n;");
+  client->DiscardAll();
+  client->Execute("FREE MEMORY;");
+  client->DiscardAll();
+
+  // now it should succeed
+  spdlog::info("Retrying the query with the memory cleaned up");
+  client->Execute(create_query);
+  if (!client->FetchOne()) {
+    LOG_FATAL("Memgraph is still out of memory");
+  }
+
+  return 0;
+}

tests/e2e/memory/workloads.yaml

@@ -0,0 +1,15 @@
+bolt_port: &bolt_port "7687"
+template_cluster: &template_cluster
+  cluster:
+    main:
+      args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
+      setup_queries: []
+      validation_queries: []
+
+workloads:
+  - name: "Memory control"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__control"
+    args: ["--bolt-port", *bolt_port, "--timeout", "180"]
+    <<: *template_cluster
+
+

tests/e2e/runner.py

@@ -12,7 +12,7 @@ PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
 BUILD_DIR = os.path.join(PROJECT_DIR, "build")
 MEMGRAPH_BINARY = os.path.join(BUILD_DIR, "memgraph")
 
-log = logging.getLogger("memgraph.tests.e2e.replication")
+log = logging.getLogger("memgraph.tests.e2e")
 
 
 def load_args():