From 4dc6400681a80847cf29af647514e7eb5bc58e1e Mon Sep 17 00:00:00 2001
From: Josip Mrden <josip.mrden@memgraph.io>
Date: Fri, 9 Feb 2024 11:02:34 +0100
Subject: [PATCH] Add tests for flags
---
src/flags/run_time_configurable.cpp | 20 +++++
src/flags/run_time_configurable.hpp | 7 ++
src/storage/v2/inmemory/storage.cpp | 9 ++-
src/storage/v2/mvcc.hpp | 40 +++++-----
src/storage/v2/transaction.hpp | 8 +-
tests/e2e/CMakeLists.txt | 1 +
tests/e2e/configuration/default_config.py | 2 +
.../maximum_deltas_restriction/CMakeLists.txt | 8 ++
.../e2e/maximum_deltas_restriction/common.py | 27 +++++++
.../maximum_deltas_restriction.py | 74 +++++++++++++++++++
.../maximum_deltas_restriction/workloads.yaml | 14 ++++
11 files changed, 188 insertions(+), 22 deletions(-)
create mode 100644 tests/e2e/maximum_deltas_restriction/CMakeLists.txt
create mode 100644 tests/e2e/maximum_deltas_restriction/common.py
create mode 100644 tests/e2e/maximum_deltas_restriction/maximum_deltas_restriction.py
create mode 100644 tests/e2e/maximum_deltas_restriction/workloads.yaml
diff --git a/src/flags/run_time_configurable.cpp b/src/flags/run_time_configurable.cpp
index a52426b5c..68bf0bc7d 100644
--- a/src/flags/run_time_configurable.cpp
+++ b/src/flags/run_time_configurable.cpp
@@ -57,6 +57,10 @@ DEFINE_bool(cartesian_product_enabled, true, "Enable cartesian product expansion
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_int64(maximum_deltas_per_transaction, -1, "Limit of deltas per transaction, default -1 (no limit)");
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_int64(maximum_delete_deltas_per_transaction, -1,
+ "Limit of delete deltas per transaction, default -1 (no limit)");
+
namespace {
// Bolt server name
constexpr auto kServerNameSettingKey = "server.name";
@@ -79,6 +83,9 @@ constexpr auto kCartesianProductEnabledGFlagsKey = "cartesian-product-enabled";
constexpr auto kMaximumDeltasPerTransactionSettingKey = "maximum-deltas-per-transaction";
constexpr auto kMaximumDeltasPerTransactionGFlagsKey = "maximum-deltas-per-transaction";
+constexpr auto kMaximumDeleteDeltasPerTransactionSettingKey = "maximum-delete-deltas-per-transaction";
+constexpr auto kMaximumDeleteDeltasPerTransactionGFlagsKey = "maximum-delete-deltas-per-transaction";
+
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
std::atomic<double> execution_timeout_sec_; // Local cache-like thing
@@ -88,6 +95,9 @@ std::atomic<bool> cartesian_product_enabled_{true}; // Local cache-like thing
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
std::atomic<int64_t> maximum_deltas_per_transaction_; // Local cache-like thing
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::atomic<int64_t> maximum_delete_deltas_per_transaction_; // Local cache-like thing
+
auto ToLLEnum(std::string_view val) {
const auto ll_enum = memgraph::flags::LogLevelToEnum(val);
if (!ll_enum) {
@@ -203,6 +213,14 @@ void Initialize() {
[&](const std::string &val) {
maximum_deltas_per_transaction_ = std::stoll(val); // Cache for faster reads
});
+
+ /*
+ * Register maximum delete deltas per transaction
+ */
+ register_flag(kMaximumDeleteDeltasPerTransactionGFlagsKey, kMaximumDeleteDeltasPerTransactionSettingKey, !kRestore,
+ [&](const std::string &val) {
+ maximum_delete_deltas_per_transaction_ = std::stoll(val); // Cache for faster reads
+ });
}
std::string GetServerName() {
@@ -218,4 +236,6 @@ bool GetCartesianProductEnabled() { return cartesian_product_enabled_; }
int64_t GetMaximumDeltasPerTransaction() { return maximum_deltas_per_transaction_; }
+int64_t GetMaximumDeleteDeltasPerTransaction() { return maximum_delete_deltas_per_transaction_; }
+
} // namespace memgraph::flags::run_time
diff --git a/src/flags/run_time_configurable.hpp b/src/flags/run_time_configurable.hpp
index 0aaebbcd4..c32154081 100644
--- a/src/flags/run_time_configurable.hpp
+++ b/src/flags/run_time_configurable.hpp
@@ -49,4 +49,11 @@ bool GetCartesianProductEnabled();
*/
int64_t GetMaximumDeltasPerTransaction();
+/**
+ * @brief Get the maximum amount of delete deltas per transaction
+ *
+ * @return int64_t
+ */
+int64_t GetMaximumDeleteDeltasPerTransaction();
+
} // namespace memgraph::flags::run_time
diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp
index 5180b9cb9..87222bade 100644
--- a/src/storage/v2/inmemory/storage.cpp
+++ b/src/storage/v2/inmemory/storage.cpp
@@ -1308,8 +1308,15 @@ Transaction InMemoryStorage::CreateTransaction(
}
auto maximum_deltas_per_transaction = flags::run_time::GetMaximumDeltasPerTransaction();
+ auto maximum_delete_deltas_per_transaction = flags::run_time::GetMaximumDeleteDeltasPerTransaction();
- return {transaction_id, start_timestamp, isolation_level, storage_mode, false, maximum_deltas_per_transaction};
+ return {transaction_id,
+ start_timestamp,
+ isolation_level,
+ storage_mode,
+ false,
+ maximum_deltas_per_transaction,
+ maximum_delete_deltas_per_transaction};
}
void InMemoryStorage::SetStorageMode(StorageMode new_storage_mode) {
diff --git a/src/storage/v2/mvcc.hpp b/src/storage/v2/mvcc.hpp
index 28b19e3e5..8eea55bea 100644
--- a/src/storage/v2/mvcc.hpp
+++ b/src/storage/v2/mvcc.hpp
@@ -14,6 +14,7 @@
#include <atomic>
#include <cstdint>
#include <optional>
+#include <unordered_set>
#include "storage/v2/property_value.hpp"
#include "storage/v2/transaction.hpp"
@@ -24,6 +25,9 @@
namespace memgraph::storage {
+const std::unordered_set<Delta::Action> delete_delta_actions{Delta::Action::ADD_IN_EDGE, Delta::Action::ADD_OUT_EDGE,
+ Delta::Action::RECREATE_OBJECT};
+
/// This function iterates through the undo buffers from an object (starting
/// from the supplied delta) and determines what deltas should be applied to get
/// the currently visible version of the object. When the function finds a delta
@@ -105,11 +109,18 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
return false;
}
-inline void IncrementDeltasChanged(Transaction *transaction) {
- transaction->deltas_changed++;
- if (transaction->max_deltas > -1 && transaction->deltas_changed > transaction->max_deltas) {
+inline void IncrementDeltasChanged(Transaction *transaction, Delta *delta) {
+ if (transaction->max_deltas > -1) {
+ transaction->deltas_changed++;
+ if (transaction->deltas_changed > transaction->max_deltas) {
+ throw utils::BasicException(
+ "You have reached the maximum number of deltas for a transaction, transaction will be rollbacked!");
+ }
+ }
+
+ if (transaction->max_delete_deltas > -1 && delete_delta_actions.contains(delta->action)) {
throw utils::BasicException(
- "You have reached the maximum number of deltas for a transaction, transaction will be rollbacked!");
+ "You have reached the maximum number of delete deltas for a transaction, transaction will be rollbacked!");
}
}
@@ -124,21 +135,12 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
}
transaction->EnsureCommitTimestampExists();
- IncrementDeltasChanged(transaction);
+ auto *delta = &transaction->deltas.use().emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
+ transaction->command_id);
- return &transaction->deltas.use().emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
- transaction->command_id);
-}
+ IncrementDeltasChanged(transaction, delta);
-inline Delta *CreateDeleteObjectDelta(Transaction *transaction, std::list<Delta> *deltas) {
- if (transaction->storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) {
- return nullptr;
- }
- transaction->EnsureCommitTimestampExists();
-
- IncrementDeltasChanged(transaction);
-
- return &deltas->emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(), transaction->command_id);
+ return delta;
}
/// TODO: what if in-memory analytical
@@ -181,11 +183,11 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
}
transaction->EnsureCommitTimestampExists();
- IncrementDeltasChanged(transaction);
-
auto delta = &transaction->deltas.use().emplace_back(std::forward<Args>(args)..., transaction->commit_timestamp.get(),
transaction->command_id);
+ IncrementDeltasChanged(transaction, delta);
+
// The operations are written in such order so that both `next` and `prev`
// chains are valid at all times. The chains must be valid at all times
// because garbage collection (which traverses the chains) is done
diff --git a/src/storage/v2/transaction.hpp b/src/storage/v2/transaction.hpp
index e9e4c599a..1d4e7ac1c 100644
--- a/src/storage/v2/transaction.hpp
+++ b/src/storage/v2/transaction.hpp
@@ -43,13 +43,15 @@ using PmrListDelta = utils::pmr::list<Delta>;
struct Transaction {
Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level,
- StorageMode storage_mode, bool edge_import_mode_active, int64_t max_deltas = -1)
+ StorageMode storage_mode, bool edge_import_mode_active, int64_t max_deltas = -1,
+ int64_t max_delete_deltas = -1)
: transaction_id(transaction_id),
start_timestamp(start_timestamp),
command_id(0),
deltas(0),
md_deltas(utils::NewDeleteResource()),
max_deltas(max_deltas),
+ max_delete_deltas(max_delete_deltas),
must_abort(false),
isolation_level(isolation_level),
storage_mode(storage_mode),
@@ -94,8 +96,10 @@ struct Transaction {
Bond<PmrListDelta> deltas;
utils::pmr::list<MetadataDelta> md_deltas;
- int64_t max_deltas{0};
+ int64_t max_deltas{-1};
+ int64_t max_delete_deltas{-1};
uint64_t deltas_changed{0};
+ uint64_t delete_deltas_changed{0};
bool must_abort{};
IsolationLevel isolation_level{};
StorageMode storage_mode{};
diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt
index a95297301..f4eaad838 100644
--- a/tests/e2e/CMakeLists.txt
+++ b/tests/e2e/CMakeLists.txt
@@ -76,6 +76,7 @@ add_subdirectory(queries)
add_subdirectory(query_modules_storage_modes)
add_subdirectory(garbage_collection)
add_subdirectory(query_planning)
+add_subdirectory(maximum_deltas_restriction)
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
add_subdirectory(high_availability_experimental)
diff --git a/tests/e2e/configuration/default_config.py b/tests/e2e/configuration/default_config.py
index 915a14d14..5940d0778 100644
--- a/tests/e2e/configuration/default_config.py
+++ b/tests/e2e/configuration/default_config.py
@@ -90,6 +90,8 @@ startup_config_dict = {
"TRACE",
"Minimum log level. Allowed values: TRACE, DEBUG, INFO, WARNING, ERROR, CRITICAL",
),
+ "maximum_deltas_per_transaction": ("-1", "-1", "Limit of deltas per transaction, default -1 (no limit)"),
+ "maximum_delete_deltas_per_transaction": ("-1", "-1", "Limit of deltas per transaction, default -1 (no limit)"),
"memory_limit": (
"0",
"0",
diff --git a/tests/e2e/maximum_deltas_restriction/CMakeLists.txt b/tests/e2e/maximum_deltas_restriction/CMakeLists.txt
new file mode 100644
index 000000000..2750905da
--- /dev/null
+++ b/tests/e2e/maximum_deltas_restriction/CMakeLists.txt
@@ -0,0 +1,8 @@
+function(copy_maximum_deltas_restriction_e2e_python_files FILE_NAME)
+ copy_e2e_python_files(maximum_deltas_restriction ${FILE_NAME})
+endfunction()
+
+copy_maximum_deltas_restriction_e2e_python_files(common.py)
+copy_maximum_deltas_restriction_e2e_python_files(maximum_deltas_restriction.py)
+
+copy_e2e_files(maximum_deltas_restriction workloads.yaml)
diff --git a/tests/e2e/maximum_deltas_restriction/common.py b/tests/e2e/maximum_deltas_restriction/common.py
new file mode 100644
index 000000000..976b649d7
--- /dev/null
+++ b/tests/e2e/maximum_deltas_restriction/common.py
@@ -0,0 +1,27 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import pytest
+from gqlalchemy import Memgraph
+
+MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER = "SET DATABASE SETTING 'maximum-deltas-per-transaction' TO '$0';"
+MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER = "SET DATABASE SETTING 'maximum-delete-deltas-per-transaction' TO '$0';"
+
+
+@pytest.fixture
+def memgraph(**kwargs) -> Memgraph:
+ memgraph = Memgraph()
+
+ yield memgraph
+
+ memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "-1"))
+ memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "-1"))
+ memgraph.drop_database()
diff --git a/tests/e2e/maximum_deltas_restriction/maximum_deltas_restriction.py b/tests/e2e/maximum_deltas_restriction/maximum_deltas_restriction.py
new file mode 100644
index 000000000..c8d7dad33
--- /dev/null
+++ b/tests/e2e/maximum_deltas_restriction/maximum_deltas_restriction.py
@@ -0,0 +1,74 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import (
+ MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER,
+ MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER,
+ memgraph,
+)
+from gqlalchemy import GQLAlchemyError
+
+CREATE_EMPTY_NODES_PLACEHOLDER = "FOREACH (i in range(1, $0) | CREATE ());"
+CREATE_FULL_NODES_PLACEHOLDER = "FOREACH (i in range(1, $0) | CREATE (:Node {id: i}));"
+DELETE_EVERYTHING_QUERY = "MATCH (n) DETACH DELETE n;"
+
+
+def test_given_no_restrictions_on_the_database_when_executing_commands_then_everything_should_pass(memgraph):
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "100"))
+ memgraph.execute(CREATE_FULL_NODES_PLACEHOLDER.replace("$0", "100"))
+ memgraph.execute(DELETE_EVERYTHING_QUERY)
+
+
+def test_given_maximum_restriction_ingestion_fails_when_inserting_more_nodes(memgraph):
+ memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
+
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "100"))
+ memgraph.execute(DELETE_EVERYTHING_QUERY)
+
+ with pytest.raises(GQLAlchemyError):
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "101"))
+
+
+def test_given_maximum_delete_restriction_deletion_fails_when_deleting_more_nodes(memgraph):
+ memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
+
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "2000"))
+
+ with pytest.raises(GQLAlchemyError):
+ memgraph.execute(DELETE_EVERYTHING_QUERY)
+
+
+def test_given_maximum_delete_restriction_deletion_fails_when_deleting_more_edges(memgraph):
+ memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
+
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "2000"))
+ memgraph.execute("CREATE (s:Supernode)")
+ memgraph.execute("MATCH (s:Supernode) MATCH (n) CREATE (s)-[:HAS]->(n)")
+
+ with pytest.raises(GQLAlchemyError):
+ memgraph.execute(DELETE_EVERYTHING_QUERY)
+
+
+def test_given_maximum_delta_restriction_fails_when_deleting_everything_on_batch_ingested_nodes(memgraph):
+ memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
+
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "90"))
+ memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "90"))
+
+ with pytest.raises(GQLAlchemyError):
+ memgraph.execute(DELETE_EVERYTHING_QUERY)
+
+
+if __name__ == "__main__":
+ sys.exit(pytest.main([__file__, "-rA"]))
diff --git a/tests/e2e/maximum_deltas_restriction/workloads.yaml b/tests/e2e/maximum_deltas_restriction/workloads.yaml
new file mode 100644
index 000000000..cedea9de4
--- /dev/null
+++ b/tests/e2e/maximum_deltas_restriction/workloads.yaml
@@ -0,0 +1,14 @@
+maximum_deltas_restriction_cluster: &maximum_deltas_restriction_cluster
+ cluster:
+ main:
+ args: ["--bolt-port", "7687", "--log-level=TRACE"]
+ log_file: "maximum_deltas_restriction.log"
+ setup_queries: []
+ validation_queries: []
+
+
+workloads:
+ - name: "Maximum deltas restriction"
+ binary: "tests/e2e/pytest_runner.sh"
+ args: ["maximum_deltas_restriction/maximum_deltas_restriction.py"]
+ <<: *maximum_deltas_restriction_cluster