From ced08fd7bc386a72ba6839cee7a7781e353faaf3 Mon Sep 17 00:00:00 2001 From: imilinovic <44698587+imilinovic@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:06:21 +0100 Subject: [PATCH 01/10] Fix GC by adding periodic jemalloc purge (#1471) --- CMakeLists.txt | 2 +- src/flags/general.cpp | 2 +- src/memory/CMakeLists.txt | 8 +-- src/storage/v2/CMakeLists.txt | 2 +- src/storage/v2/inmemory/storage.cpp | 11 +++- src/storage/v2/inmemory/storage.hpp | 1 + tests/e2e/CMakeLists.txt | 1 + tests/e2e/garbage_collection/CMakeLists.txt | 7 +++ tests/e2e/garbage_collection/common.py | 25 +++++++++ tests/e2e/garbage_collection/conftest.py | 21 ++++++++ tests/e2e/garbage_collection/gc_periodic.py | 59 +++++++++++++++++++++ tests/e2e/garbage_collection/workloads.yaml | 19 +++++++ 12 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/garbage_collection/CMakeLists.txt create mode 100644 tests/e2e/garbage_collection/common.py create mode 100644 tests/e2e/garbage_collection/conftest.py create mode 100644 tests/e2e/garbage_collection/gc_periodic.py create mode 100644 tests/e2e/garbage_collection/workloads.yaml diff --git a/CMakeLists.txt b/CMakeLists.txt index db39a4547..a5ad2612a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,7 +292,7 @@ if (MG_ENTERPRISE) add_definitions(-DMG_ENTERPRISE) endif() -set(ENABLE_JEMALLOC ON) +option(ENABLE_JEMALLOC "Use jemalloc" ON) if (ASAN) message(WARNING "Disabling jemalloc as it doesn't work well with ASAN") diff --git a/src/flags/general.cpp b/src/flags/general.cpp index eb8fae589..be060c52d 100644 --- a/src/flags/general.cpp +++ b/src/flags/general.cpp @@ -65,7 +65,7 @@ DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed i // Storage flags. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).", - FLAG_IN_RANGE(1, 24 * 3600)); + FLAG_IN_RANGE(1, 24UL * 3600)); // NOTE: The `storage_properties_on_edges` flag must be the same here and in // `mg_import_csv`. If you change it, make sure to change it there as well. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/memory/CMakeLists.txt b/src/memory/CMakeLists.txt index aadbbe23c..e975c1d5c 100644 --- a/src/memory/CMakeLists.txt +++ b/src/memory/CMakeLists.txt @@ -3,14 +3,14 @@ set(memory_src_files global_memory_control.cpp query_memory_control.cpp) - - -find_package(jemalloc REQUIRED) - add_library(mg-memory STATIC ${memory_src_files}) target_link_libraries(mg-memory mg-utils fmt) +message(STATUS "ENABLE_JEMALLOC: ${ENABLE_JEMALLOC}") if (ENABLE_JEMALLOC) + find_package(jemalloc REQUIRED) target_link_libraries(mg-memory Jemalloc::Jemalloc ${CMAKE_DL_LIBS}) target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1) +else() + target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=0) endif() diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 147684c54..9f6d8d4d7 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -41,4 +41,4 @@ add_library(mg-storage-v2 STATIC replication/replication_storage_state.cpp inmemory/replication/replication_client.cpp ) -target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events) +target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory) diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 9d2bc7a65..d0d1dd071 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -11,6 +11,7 @@ #include "storage/v2/inmemory/storage.hpp" #include "dbms/constants.hpp" +#include "memory/global_memory_control.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/metadata_delta.hpp" @@ -101,8 +102,13 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) "those files into a .backup directory inside the storage directory."); } } + if (config_.gc.type == Config::Gc::Type::PERIODIC) { - gc_runner_.Run("Storage GC", config_.gc.interval, [this] { this->CollectGarbage(); }); + // TODO: move out of storage have one global gc_runner_ + gc_runner_.Run("Storage GC", config_.gc.interval, [this] { + this->FreeMemory(std::unique_lock{main_lock_, std::defer_lock}); + }); + gc_jemalloc_runner_.Run("Jemalloc GC", config_.gc.interval, [] { memory::PurgeUnusedMemory(); }); } if (timestamp_ == kTimestampInitialId) { commit_log_.emplace(); @@ -116,6 +122,7 @@ InMemoryStorage::InMemoryStorage(Config config) : InMemoryStorage(config, Storag InMemoryStorage::~InMemoryStorage() { if (config_.gc.type == Config::Gc::Type::PERIODIC) { gc_runner_.Stop(); + gc_jemalloc_runner_.Stop(); } { // Stop replication (Stop all clients or stop the REPLICA server) @@ -1210,7 +1217,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_ main_lock_.lock_shared(); } } else { - MG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_"); + DMG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_"); } utils::OnScopeExit lock_releaser{[&] { diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 1d16eadf1..bfb445332 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -418,6 +418,7 @@ class InMemoryStorage final : public Storage { std::optional commit_log_; utils::Scheduler gc_runner_; + utils::Scheduler gc_jemalloc_runner_; std::mutex gc_lock_; using BondPmrLd = Bond>; diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 71d80b7ed..28fe94559 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -71,6 +71,7 @@ add_subdirectory(query_modules) add_subdirectory(constraints) add_subdirectory(inspect_query) add_subdirectory(queries) +add_subdirectory(garbage_collection) copy_e2e_python_files(pytest_runner pytest_runner.sh "") copy_e2e_python_files(x x.sh "") diff --git a/tests/e2e/garbage_collection/CMakeLists.txt b/tests/e2e/garbage_collection/CMakeLists.txt new file mode 100644 index 000000000..690edf344 --- /dev/null +++ b/tests/e2e/garbage_collection/CMakeLists.txt @@ -0,0 +1,7 @@ +function(garbage_collection_e2e_python_files FILE_NAME) + copy_e2e_python_files(garbage_collection ${FILE_NAME}) +endfunction() + +garbage_collection_e2e_python_files(common.py) +garbage_collection_e2e_python_files(conftest.py) +garbage_collection_e2e_python_files(gc_periodic.py) diff --git a/tests/e2e/garbage_collection/common.py b/tests/e2e/garbage_collection/common.py new file mode 100644 index 000000000..dedf18dc3 --- /dev/null +++ b/tests/e2e/garbage_collection/common.py @@ -0,0 +1,25 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import typing + +import mgclient + + +def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]: + cursor.execute(query, params) + return cursor.fetchall() + + +def connect(**kwargs) -> mgclient.Connection: + connection = mgclient.connect(host="localhost", port=7687, **kwargs) + connection.autocommit = True + return connection diff --git a/tests/e2e/garbage_collection/conftest.py b/tests/e2e/garbage_collection/conftest.py new file mode 100644 index 000000000..a4ec62c9f --- /dev/null +++ b/tests/e2e/garbage_collection/conftest.py @@ -0,0 +1,21 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import pytest +from common import connect, execute_and_fetch_all + + +@pytest.fixture(autouse=True) +def connection(): + connection = connect() + yield connection + cursor = connection.cursor() + execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n") diff --git a/tests/e2e/garbage_collection/gc_periodic.py b/tests/e2e/garbage_collection/gc_periodic.py new file mode 100644 index 000000000..6903960be --- /dev/null +++ b/tests/e2e/garbage_collection/gc_periodic.py @@ -0,0 +1,59 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import re +import sys +import time + +import pytest +from common import execute_and_fetch_all + + +def remove_non_numeric_suffix(text): + match = re.search(r"\D*$", text) + if match: + non_numeric_suffix = match.group(0) + return text[: -len(non_numeric_suffix)] + else: + return text + + +def get_memory_from_list(list): + for list_item in list: + if list_item[0] == "memory_tracked": + return float(remove_non_numeric_suffix(list_item[1])) + return None + + +def get_memory(cursor): + return get_memory_from_list(execute_and_fetch_all(cursor, "SHOW STORAGE INFO")) + + +def test_gc_periodic(connection): + """ + This test checks that periodic gc works. + It does so by checking that the allocated memory is lowered by at least 1/4 of the memory allocated by creating nodes. + If we choose a number a high number the test will become flaky because the memory only gets fully cleared after a while + due to jemalloc holding some memory for a while. If we'd wait for jemalloc to fully release the memory the test would take too long. + """ + cursor = connection.cursor() + + memory_pre_creation = get_memory(cursor) + execute_and_fetch_all(cursor, "UNWIND range(1, 1000) AS index CREATE (:Node);") + memory_after_creation = get_memory(cursor) + time.sleep(2) + memory_after_gc = get_memory(cursor) + + assert memory_after_gc < memory_pre_creation + (memory_after_creation - memory_pre_creation) / 4 * 3 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/garbage_collection/workloads.yaml b/tests/e2e/garbage_collection/workloads.yaml new file mode 100644 index 000000000..395ba83d9 --- /dev/null +++ b/tests/e2e/garbage_collection/workloads.yaml @@ -0,0 +1,19 @@ +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--storage-gc-cycle-sec=2" + +in_memory_cluster: &in_memory_cluster + cluster: + main: + args: *args + log_file: "garbage_collection-e2e.log" + setup_queries: [] + validation_queries: [] + +workloads: + - name: "Garbage collection" + binary: "tests/e2e/pytest_runner.sh" + args: ["garbage_collection/gc_periodic.py"] + <<: *in_memory_cluster From c037cddb0ec57ca0cdebfd7bd0c1af024ea66164 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Tue, 14 Nov 2023 23:23:06 +0100 Subject: [PATCH 02/10] Add granular index and constraint recovery info (#1480) --- src/dbms/inmemory/replication_handlers.cpp | 3 +- src/storage/v2/durability/durability.cpp | 34 +++++++++++++++------- src/storage/v2/durability/durability.hpp | 2 +- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/dbms/inmemory/replication_handlers.cpp b/src/dbms/inmemory/replication_handlers.cpp index b50163c55..ce1f6da20 100644 --- a/src/dbms/inmemory/replication_handlers.cpp +++ b/src/dbms/inmemory/replication_handlers.cpp @@ -220,7 +220,8 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle spdlog::trace("Recovering indices and constraints from snapshot."); storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_, - &storage->constraints_, &storage->vertices_); + &storage->constraints_, &storage->vertices_, + storage->name_id_mapper_.get()); } catch (const storage::durability::RecoveryFailure &e) { LOG_FATAL("Couldn't load the snapshot because of: {}", e.what()); } diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index decbfd14a..1240bd52e 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -130,16 +130,17 @@ std::optional> GetWalFiles(const std::filesystem: // recovery process. void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints, utils::SkipList *vertices, + NameIdMapper *name_id_mapper, const std::optional ¶llel_exec_info) { spdlog::info("Recreating indices from metadata."); // Recover label indices. spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size()); auto *mem_label_index = static_cast(indices->label_index_.get()); for (const auto &item : indices_constraints.indices.label) { - if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) + if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) { throw RecoveryFailure("The label index must be created here!"); - - spdlog::info("A label index is recreated from metadata."); + } + spdlog::info("Index on :{} is recreated from metadata", name_id_mapper->IdToName(item.AsUint())); } spdlog::info("Label indices are recreated."); @@ -148,7 +149,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Recreating {} label index statistics from metadata.", indices_constraints.indices.label_stats.size()); for (const auto &item : indices_constraints.indices.label_stats) { mem_label_index->SetIndexStats(item.first, item.second); - spdlog::info("A label index statistics is recreated from metadata."); + spdlog::info("Statistics for index on :{} are recreated from metadata", + name_id_mapper->IdToName(item.first.AsUint())); } spdlog::info("Label indices statistics are recreated."); @@ -159,7 +161,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ for (const auto &item : indices_constraints.indices.label_property) { if (!mem_label_property_index->CreateIndex(item.first, item.second, vertices->access(), parallel_exec_info)) throw RecoveryFailure("The label+property index must be created here!"); - spdlog::info("A label+property index is recreated from metadata."); + spdlog::info("Index on :{}({}) is recreated from metadata", name_id_mapper->IdToName(item.first.AsUint()), + name_id_mapper->IdToName(item.second.AsUint())); } spdlog::info("Label+property indices are recreated."); @@ -171,7 +174,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ const auto property_id = item.second.first; const auto &stats = item.second.second; mem_label_property_index->SetIndexStats({label_id, property_id}, stats); - spdlog::info("A label+property index statistics is recreated from metadata."); + spdlog::info("Statistics for index on :{}({}) are recreated from metadata", + name_id_mapper->IdToName(label_id.AsUint()), name_id_mapper->IdToName(property_id.AsUint())); } spdlog::info("Label+property indices statistics are recreated."); @@ -191,8 +195,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ } constraints->existence_constraints_->InsertConstraint(label, property); - - spdlog::info("A existence constraint is recreated from metadata."); + spdlog::info("Existence constraint on :{}({}) is recreated from metadata", name_id_mapper->IdToName(label.AsUint()), + name_id_mapper->IdToName(property.AsUint())); } spdlog::info("Existence constraints are recreated from metadata."); @@ -203,7 +207,15 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ auto ret = mem_unique_constraints->CreateConstraint(item.first, item.second, vertices->access()); if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) throw RecoveryFailure("The unique constraint must be created here!"); - spdlog::info("A unique constraint is recreated from metadata."); + + std::vector property_names; + property_names.reserve(item.second.size()); + for (const auto &prop : item.second) { + property_names.emplace_back(name_id_mapper->IdToName(prop.AsUint())); + } + const auto property_names_joined = utils::Join(property_names, ","); + spdlog::info("Unique constraint on :{}({}) is recreated from metadata", + name_id_mapper->IdToName(item.first.AsUint()), property_names_joined); } spdlog::info("Unique constraints are recreated from metadata."); spdlog::info("Constraints are recreated from metadata."); @@ -270,7 +282,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) : std::nullopt; - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info); + RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); return recovered_snapshot->recovery_info; } } else { @@ -402,7 +414,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di ? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count)) : std::nullopt; - RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info); + RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info); memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us, std::chrono::duration_cast(timer.Elapsed()).count()); diff --git a/src/storage/v2/durability/durability.hpp b/src/storage/v2/durability/durability.hpp index 8b735f02a..8bb1223c4 100644 --- a/src/storage/v2/durability/durability.hpp +++ b/src/storage/v2/durability/durability.hpp @@ -104,7 +104,7 @@ using ParallelizedIndexCreationInfo = /// @throw RecoveryFailure void RecoverIndicesAndConstraints( const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints, - utils::SkipList *vertices, + utils::SkipList *vertices, NameIdMapper *name_id_mapper, const std::optional ¶llel_exec_info = std::nullopt); /// Recovers data either from a snapshot and/or WAL files. From d3f4c3536299ea2da2e39123b087500553150411 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:42:04 +0100 Subject: [PATCH 03/10] Add OOM enabler for MG procedure (#1401) --- src/query/plan/operator.cpp | 2 + src/query/plan/rule_based_planner.hpp | 9 +- src/query/procedure/mg_procedure_impl.cpp | 3 + tests/e2e/memory/CMakeLists.txt | 9 +- ..._limit_global_multi_thread_proc_create.cpp | 67 +++++++++++++ .../memory_limit_global_thread_alloc_proc.cpp | 68 +++++++++++++ .../procedure_memory_limit_multi_proc.cpp | 7 +- tests/e2e/memory/procedures/CMakeLists.txt | 9 +- ..._memory_limit_multi_thread_create_proc.cpp | 95 +++++++++++++++++++ .../global_memory_limit_thread_proc.cpp | 92 ++++++++++++++++++ .../memory/procedures/proc_memory_limit.cpp | 3 + .../query_memory_limit_proc_multi_thread.cpp | 3 +- tests/e2e/memory/workloads.yaml | 15 ++- 13 files changed, 368 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp create mode 100644 tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp create mode 100644 tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp create mode 100644 tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 238638737..52e15e928 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -57,6 +57,7 @@ #include "utils/likely.hpp" #include "utils/logging.hpp" #include "utils/memory.hpp" +#include "utils/memory_tracker.hpp" #include "utils/message.hpp" #include "utils/on_scope_exit.hpp" #include "utils/pmr/deque.hpp" @@ -4859,6 +4860,7 @@ class CallProcedureCursor : public Cursor { result_signature_size_ = result_->signature->size(); result_->signature = nullptr; if (result_->error_msg) { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker; throw QueryRuntimeException("{}: {}", self_->procedure_name_, *result_->error_msg); } result_row_it_ = result_->rows.begin(); diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index afa060b9d..bdac76a93 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -176,7 +176,10 @@ class RuleBasedPlanner { PlanResult Plan(const QueryParts &query_parts) { auto &context = *context_; std::unique_ptr final_plan; - + // procedures need to start from 1 + // due to swapping mechanism of procedure + // tracking + uint64_t procedure_id = 1; for (const auto &query_part : query_parts.query_parts) { std::unique_ptr input_op; @@ -186,10 +189,6 @@ class RuleBasedPlanner { uint64_t merge_id = 0; uint64_t subquery_id = 0; - // procedures need to start from 1 - // due to swapping mechanism of procedure - // tracking - uint64_t procedure_id = 1; for (const auto &clause : single_query_part.remaining_clauses) { MG_ASSERT(!utils::IsSubtype(*clause, Match::kType), "Unexpected Match in remaining clauses"); diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index 2a657aeb3..2a176f2ed 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -38,6 +38,7 @@ #include "utils/logging.hpp" #include "utils/math.hpp" #include "utils/memory.hpp" +#include "utils/memory_tracker.hpp" #include "utils/string.hpp" #include "utils/temporal.hpp" #include "utils/variant_helpers.hpp" @@ -158,6 +159,7 @@ template [[nodiscard]] mgp_error WrapExceptions(TFunc &&func, Args &&...args) noexcept { static_assert(sizeof...(args) <= 1, "WrapExceptions should have only one or zero parameter!"); try { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_enabler; WrapExceptionsHelper(std::forward(func), std::forward(args)...); } catch (const DeletedObjectException &neoe) { spdlog::error("Deleted object error during mg API call: {}", neoe.what()); @@ -1544,6 +1546,7 @@ mgp_error mgp_duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory mgp_error mgp_result_set_error_msg(mgp_result *res, const char *msg) { return WrapExceptions([=] { + memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker{}; auto *memory = res->rows.get_allocator().GetMemoryResource(); res->error_msg.emplace(msg, memory); }); diff --git a/tests/e2e/memory/CMakeLists.txt b/tests/e2e/memory/CMakeLists.txt index 327f09106..3c4cdc279 100644 --- a/tests/e2e/memory/CMakeLists.txt +++ b/tests/e2e/memory/CMakeLists.txt @@ -2,6 +2,8 @@ add_subdirectory(procedures) find_package(gflags REQUIRED) +# Global memory limit + add_executable(memgraph__e2e__memory__control memory_control.cpp) target_link_libraries(memgraph__e2e__memory__control gflags mgclient mg-utils mg-io Threads::Threads) @@ -20,6 +22,12 @@ target_link_libraries(memgraph__e2e__memory__limit_accumulation gflags mgclient add_executable(memgraph__e2e__memory__limit_edge_create memory_limit_edge_create.cpp) target_link_libraries(memgraph__e2e__memory__limit_edge_create gflags mgclient mg-utils mg-io) +add_executable(memgraph__e2e__memory_limit_global_multi_thread_proc_create memory_limit_global_multi_thread_proc_create.cpp) +target_link_libraries(memgraph__e2e__memory_limit_global_multi_thread_proc_create gflags mgclient mg-utils mg-io) + +add_executable(memgraph__e2e__memory_limit_global_thread_alloc_proc memory_limit_global_thread_alloc_proc.cpp) +target_link_libraries(memgraph__e2e__memory_limit_global_thread_alloc_proc gflags mgclient mg-utils mg-io) + # Query memory limit tests add_executable(memgraph__e2e__memory__limit_query_alloc_proc_multi_thread query_memory_limit_proc_multi_thread.cpp) @@ -34,7 +42,6 @@ target_link_libraries(memgraph__e2e__memory__limit_query_alloc_proc gflags mgcli add_executable(memgraph__e2e__memory__limit_query_alloc_create_multi_thread query_memory_limit_multi_thread.cpp) target_link_libraries(memgraph__e2e__memory__limit_query_alloc_create_multi_thread gflags mgclient mg-utils mg-io Threads::Threads) - # Procedure memory limit tests add_executable(memgraph__e2e__procedure_memory_limit procedure_memory_limit.cpp) diff --git a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp new file mode 100644 index 000000000..2132fbb16 --- /dev/null +++ b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp @@ -0,0 +1,67 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include "utils/logging.hpp" +#include "utils/timer.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); +DEFINE_uint64(timeout, 120, "Timeout seconds"); +DEFINE_bool(multi_db, false, "Run test in multi db environment"); + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread Create For Local Allocators"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + mg::Client::Init(); + + auto client = + mg::Client::Connect({.host = "127.0.0.1", .port = static_cast(FLAGS_bolt_port), .use_ssl = false}); + if (!client) { + LOG_FATAL("Failed to connect!"); + } + + if (FLAGS_multi_db) { + client->Execute("CREATE DATABASE clean;"); + client->DiscardAll(); + client->Execute("USE DATABASE clean;"); + client->DiscardAll(); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + } + + bool error{false}; + try { + client->Execute( + "CALL libglobal_memory_limit_multi_thread_create_proc.multi_create() PROCEDURE MEMORY UNLIMITED YIELD " + "allocated_all RETURN allocated_all " + "QUERY MEMORY LIMIT 50MB;"); + auto result_rows = client->FetchAll(); + if (result_rows) { + auto row = *result_rows->begin(); + error = row[0].ValueBool() == false; + } + + } catch (const std::exception &e) { + error = true; + } + + MG_ASSERT(error, "Error should have happend"); + + return 0; +} diff --git a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp new file mode 100644 index 000000000..0c2eb1ee6 --- /dev/null +++ b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp @@ -0,0 +1,68 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include + +#include "utils/logging.hpp" +#include "utils/timer.hpp" + +DEFINE_uint64(bolt_port, 7687, "Bolt port"); +DEFINE_uint64(timeout, 120, "Timeout seconds"); +DEFINE_bool(multi_db, false, "Run test in multi db environment"); + +// Test checks path of throwing error from different thread +// than main thread which started test +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread For Procedures For Local Allocators"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + mg::Client::Init(); + + auto client = + mg::Client::Connect({.host = "127.0.0.1", .port = static_cast(FLAGS_bolt_port), .use_ssl = false}); + if (!client) { + LOG_FATAL("Failed to connect!"); + } + + if (FLAGS_multi_db) { + client->Execute("CREATE DATABASE clean;"); + client->DiscardAll(); + client->Execute("USE DATABASE clean;"); + client->DiscardAll(); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + } + + bool error{false}; + try { + client->Execute( + "CALL libglobal_memory_limit_thread_proc.thread() YIELD allocated_all RETURN allocated_all QUERY MEMORY LIMIT " + "100MB;"); + auto result_rows = client->FetchAll(); + if (result_rows) { + auto row = *result_rows->begin(); + error = row[0].ValueBool() == false; + } + + } catch (const std::exception &e) { + error = true; + } + + MG_ASSERT(error, "Error should have happend"); + + return 0; +} diff --git a/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp b/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp index 118ff41ce..f850ec6c2 100644 --- a/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp +++ b/tests/e2e/memory/procedure_memory_limit_multi_proc.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "utils/logging.hpp" @@ -53,11 +54,9 @@ int main(int argc, char **argv) { "CALL libproc_memory_limit.alloc_32_mib() PROCEDURE MEMORY LIMIT 10MB YIELD allocated AS allocated_2 RETURN " "allocated_1, allocated_2"); auto result_rows = client->FetchAll(); - if (result_rows) { - auto row = *result_rows->begin(); - test_passed = row[0].ValueBool() == true && row[0].ValueBool() == false; + if (result_rows && result_rows->empty()) { + test_passed = true; } - } catch (const std::exception &e) { test_passed = true; } diff --git a/tests/e2e/memory/procedures/CMakeLists.txt b/tests/e2e/memory/procedures/CMakeLists.txt index 84c56f414..df7acee31 100644 --- a/tests/e2e/memory/procedures/CMakeLists.txt +++ b/tests/e2e/memory/procedures/CMakeLists.txt @@ -4,16 +4,21 @@ target_include_directories(global_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/inclu add_library(global_memory_limit_proc SHARED global_memory_limit_proc.c) target_include_directories(global_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) - add_library(query_memory_limit_proc_multi_thread SHARED query_memory_limit_proc_multi_thread.cpp) target_include_directories(query_memory_limit_proc_multi_thread PRIVATE ${CMAKE_SOURCE_DIR}/include) target_link_libraries(query_memory_limit_proc_multi_thread mg-utils) - add_library(query_memory_limit_proc SHARED query_memory_limit_proc.cpp) target_include_directories(query_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) target_link_libraries(query_memory_limit_proc mg-utils) +add_library(global_memory_limit_thread_proc SHARED global_memory_limit_thread_proc.cpp) +target_include_directories(global_memory_limit_thread_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(global_memory_limit_thread_proc mg-utils) + +add_library(global_memory_limit_multi_thread_create_proc SHARED global_memory_limit_multi_thread_create_proc.cpp) +target_include_directories(global_memory_limit_multi_thread_create_proc PRIVATE ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(global_memory_limit_multi_thread_create_proc mg-utils) add_library(proc_memory_limit SHARED proc_memory_limit.cpp) target_include_directories(proc_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/include) diff --git a/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp b/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp new file mode 100644 index 000000000..2ccaac631 --- /dev/null +++ b/tests/e2e/memory/procedures/global_memory_limit_multi_thread_create_proc.cpp @@ -0,0 +1,95 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mg_procedure.h" +#include "mgp.hpp" +#include "utils/on_scope_exit.hpp" + +// change communication between threads with feature and promise +std::atomic created_vertices{0}; +constexpr int num_vertices_per_thread{100'000}; +constexpr int num_threads{2}; + +void CallCreate(mgp_graph *graph, mgp_memory *memory) { + [[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph); + for (int i = 0; i < num_vertices_per_thread; i++) { + struct mgp_vertex *vertex{nullptr}; + auto enum_error = mgp_graph_create_vertex(graph, memory, &vertex); + if (enum_error != mgp_error::MGP_ERROR_NO_ERROR) { + break; + } + created_vertices.fetch_add(1, std::memory_order_acq_rel); + } + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); +} + +void AllocFunc(mgp_graph *graph, mgp_memory *memory) { + try { + CallCreate(graph, memory); + } catch (const std::exception &e) { + return; + } +} + +void MultiCreate(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto arguments = mgp::List(args); + const auto record_factory = mgp::RecordFactory(result); + try { + std::vector threads; + + for (int i = 0; i < 2; i++) { + threads.emplace_back(AllocFunc, memgraph_graph, memory); + } + + for (int i = 0; i < num_threads; i++) { + threads[i].join(); + } + if (created_vertices.load(std::memory_order_acquire) != num_vertices_per_thread * num_threads) { + record_factory.SetErrorMessage("Unable to allocate"); + return; + } + + auto new_record = record_factory.NewRecord(); + new_record.Insert("allocated_all", + created_vertices.load(std::memory_order_acquire) == num_vertices_per_thread * num_threads); + } catch (std::exception &e) { + record_factory.SetErrorMessage(e.what()); + } +} + +extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { + try { + mgp::MemoryDispatcherGuard guard{memory}; + + AddProcedure(MultiCreate, std::string("multi_create").c_str(), mgp::ProcedureType::Write, {}, + {mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory); + + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp b/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp new file mode 100644 index 000000000..95bf19f9d --- /dev/null +++ b/tests/e2e/memory/procedures/global_memory_limit_thread_proc.cpp @@ -0,0 +1,92 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mg_procedure.h" +#include "mgp.hpp" +#include "utils/on_scope_exit.hpp" + +enum mgp_error Alloc(mgp_memory *memory, void *ptr) { + const size_t mb_size_512 = 1 << 29; + + return mgp_alloc(memory, mb_size_512, (void **)(&ptr)); +} + +// change communication between threads with feature and promise +std::atomic num_allocations{0}; +void *ptr_; + +void AllocFunc(mgp_memory *memory, mgp_graph *graph) { + try { + [[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph); + enum mgp_error alloc_err { mgp_error::MGP_ERROR_NO_ERROR }; + alloc_err = Alloc(memory, ptr_); + if (alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) { + num_allocations.fetch_add(1, std::memory_order_relaxed); + } + if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) { + assert(false); + } + } catch (const std::exception &e) { + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); + assert(false); + } + [[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph); +} + +void Thread(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto arguments = mgp::List(args); + const auto record_factory = mgp::RecordFactory(result); + num_allocations.store(0, std::memory_order_relaxed); + try { + std::thread thread{AllocFunc, memory, memgraph_graph}; + + thread.join(); + + if (ptr_ != nullptr) { + mgp_free(memory, ptr_); + } + + auto new_record = record_factory.NewRecord(); + + new_record.Insert("allocated_all", num_allocations.load(std::memory_order_relaxed) == 1); + } catch (std::exception &e) { + record_factory.SetErrorMessage(e.what()); + } +} + +extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { + try { + mgp::memory = memory; + + AddProcedure(Thread, std::string("thread").c_str(), mgp::ProcedureType::Read, {}, + {mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory); + + } catch (const std::exception &e) { + return 1; + } + + return 0; +} + +extern "C" int mgp_shutdown_module() { return 0; } diff --git a/tests/e2e/memory/procedures/proc_memory_limit.cpp b/tests/e2e/memory/procedures/proc_memory_limit.cpp index 9f36dfaa4..d78407222 100644 --- a/tests/e2e/memory/procedures/proc_memory_limit.cpp +++ b/tests/e2e/memory/procedures/proc_memory_limit.cpp @@ -75,6 +75,9 @@ void Alloc_32_MiB(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, }}; const enum mgp_error alloc_err = Alloc_32(memory, ptr); + if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) { + record_factory.SetErrorMessage("Unable to allocate"); + } auto new_record = record_factory.NewRecord(); new_record.Insert("allocated", alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE); } catch (std::exception &e) { diff --git a/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp b/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp index ffc509ff3..0a1d8f125 100644 --- a/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp +++ b/tests/e2e/memory/procedures/query_memory_limit_proc_multi_thread.cpp @@ -13,7 +13,7 @@ #include #include #include -#include + #include #include #include @@ -22,6 +22,7 @@ #include #include "mg_procedure.h" +#include "mgp.hpp" #include "utils/on_scope_exit.hpp" enum mgp_error Alloc(void *ptr) { diff --git a/tests/e2e/memory/workloads.yaml b/tests/e2e/memory/workloads.yaml index e84faccf0..d826175ed 100644 --- a/tests/e2e/memory/workloads.yaml +++ b/tests/e2e/memory/workloads.yaml @@ -144,6 +144,7 @@ workloads: binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create_multi_thread" args: ["--bolt-port", *bolt_port] <<: *in_memory_query_limit_cluster + - name: "Memory control for detach delete" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_delete" args: ["--bolt-port", *bolt_port] @@ -174,6 +175,18 @@ workloads: args: ["--bolt-port", *bolt_port] <<: *disk_450_MiB_limit_cluster + - name: "Memory control for create from multi thread proc create" + binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_multi_thread_proc_create" + proc: "tests/e2e/memory/procedures/" + args: ["--bolt-port", *bolt_port] + <<: *in_memory_cluster + + - name: "Memory control for memory limit global thread alloc" + binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_thread_alloc_proc" + proc: "tests/e2e/memory/procedures/" + args: ["--bolt-port", *bolt_port] + <<: *in_memory_cluster + - name: "Procedure memory control for single procedure" binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit" proc: "tests/e2e/memory/procedures/" @@ -181,7 +194,7 @@ workloads: <<: *in_memory_limited_global_limit_cluster - name: "Procedure memory control for multiple procedures" - binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit" + binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit_multi_proc" proc: "tests/e2e/memory/procedures/" args: ["--bolt-port", *bolt_port] <<: *in_memory_limited_global_limit_cluster From 645568a75bc55411c61f1499d6a810c79bebba44 Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:01:44 +0100 Subject: [PATCH 04/10] Remove default memory limit on procedures (#1506) * remove default limit on procedures * fix bug on GraphQL also --- .../frontend/ast/cypher_main_visitor.cpp | 10 ++++---- tests/unit/cypher_main_visitor.cpp | 23 ------------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 6b75061fb..4bf7f36fd 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -1216,10 +1216,6 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur call_proc->memory_limit_ = memory_limit_info->first; call_proc->memory_scale_ = memory_limit_info->second; } - } else { - // Default to 100 MB - call_proc->memory_limit_ = storage_->Create(TypedValue(100)); - call_proc->memory_scale_ = 1024U * 1024U; } const auto &maybe_found = @@ -1240,11 +1236,13 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); } } - call_proc->is_write_ = maybe_found->second->info.is_write; + if (maybe_found) { + call_proc->is_write_ = maybe_found->second->info.is_write; + } auto *yield_ctx = ctx->yieldProcedureResults(); if (!yield_ctx) { - if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) { + if ((maybe_found && !maybe_found->second->results.empty()) && !call_proc->void_procedure_) { throw SemanticException( "CALL without YIELD may only be used on procedures which do not " "return any result fields."); diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index acbd1fe41..1d7f236a2 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -2833,18 +2833,6 @@ TEST_P(CypherMainVisitorTest, DumpDatabase) { ASSERT_TRUE(query); } -namespace { -template -void CheckCallProcedureDefaultMemoryLimit(const TAst &ast, const CallProcedure &call_proc) { - // Should be 100 MB - auto *literal = dynamic_cast(call_proc.memory_limit_); - ASSERT_TRUE(literal); - TypedValue value(literal->value_); - ASSERT_TRUE(TypedValue::BoolEqual{}(value, TypedValue(100))); - ASSERT_EQ(call_proc.memory_scale_, 1024 * 1024); -} -} // namespace - TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) { AddProc(*mock_module_with_dots_in_name, "proc", {}, {"res"}, ProcedureType::WRITE); auto &ast_generator = *GetParam(); @@ -2868,7 +2856,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) { @@ -2894,7 +2881,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) { @@ -2926,7 +2912,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) { std::vector expected_names{"fst", "field-with-dashes", "last_field"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); }; check_proc(ProcedureType::READ); check_proc(ProcedureType::WRITE); @@ -2959,7 +2944,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldAliasedFields) { ASSERT_EQ(identifier_names, aliased_names); std::vector field_names{"fst", "snd", "thrd"}; ASSERT_EQ(call_proc->result_fields_, field_names); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) { @@ -2986,7 +2970,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) { std::vector expected_names{"res"}; ASSERT_EQ(identifier_names, expected_names); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) { @@ -3008,7 +2991,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) { } ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable")); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) { @@ -3033,7 +3015,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) { } ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable")); ASSERT_EQ(identifier_names, call_proc->result_fields_); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) { @@ -3049,7 +3030,6 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) { ASSERT_TRUE(call_proc->arguments_.empty()); ASSERT_TRUE(call_proc->result_fields_.empty()); ASSERT_TRUE(call_proc->result_identifiers_.empty()); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } TEST_P(CypherMainVisitorTest, CallProcedureWithMemoryLimitWithoutYield) { @@ -3183,7 +3163,6 @@ void CheckParsedCallProcedure(const CypherQuery &query, Base &ast_generator, EXPECT_EQ(identifier_names, args_as_str); EXPECT_EQ(identifier_names, call_proc->result_fields_); ASSERT_EQ(call_proc->is_write_, type == ProcedureType::WRITE); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); }; } // namespace @@ -3577,7 +3556,6 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) { auto *single_query = query->single_query_; ASSERT_EQ(single_query->clauses_.size(), 2U); auto *call_proc = dynamic_cast(single_query->clauses_[0]); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } { @@ -3638,7 +3616,6 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) { auto *single_query = query->single_query_; ASSERT_EQ(single_query->clauses_.size(), 1U); auto *call_proc = dynamic_cast(single_query->clauses_[0]); - CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc); } } From 6053a91ef86bfa3fb321f99d7c3d5ba2bce5db69 Mon Sep 17 00:00:00 2001 From: imilinovic <44698587+imilinovic@users.noreply.github.com> Date: Fri, 17 Nov 2023 23:06:46 +0100 Subject: [PATCH 05/10] Fix flaky GC test (#1521) --- tests/e2e/garbage_collection/gc_periodic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/garbage_collection/gc_periodic.py b/tests/e2e/garbage_collection/gc_periodic.py index 6903960be..a93846a68 100644 --- a/tests/e2e/garbage_collection/gc_periodic.py +++ b/tests/e2e/garbage_collection/gc_periodic.py @@ -49,7 +49,7 @@ def test_gc_periodic(connection): memory_pre_creation = get_memory(cursor) execute_and_fetch_all(cursor, "UNWIND range(1, 1000) AS index CREATE (:Node);") memory_after_creation = get_memory(cursor) - time.sleep(2) + time.sleep(5) memory_after_gc = get_memory(cursor) assert memory_after_gc < memory_pre_creation + (memory_after_creation - memory_pre_creation) / 4 * 3 From d03fafcef6fae57c7e456372f64ea227fb84b316 Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 20 Nov 2023 11:52:17 +0100 Subject: [PATCH 06/10] Aggregations return empty result when used with group by (#1531) --- src/query/plan/operator.cpp | 8 +---- src/query/plan/operator.hpp | 2 -- .../memgraph_V1/features/aggregations.feature | 4 +-- .../features/aggregations.feature | 4 +-- .../unit/query_plan_accumulate_aggregate.cpp | 34 ++++++------------- 5 files changed, 13 insertions(+), 39 deletions(-) diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 52e15e928..b68810ad7 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -3464,7 +3464,7 @@ class AggregateCursor : public Cursor { SCOPED_PROFILE_OP_BY_REF(self_); if (!pulled_all_input_) { - if (!ProcessAll(&frame, &context) && self_.AreAllAggregationsForCollecting()) return false; + if (!ProcessAll(&frame, &context) && !self_.group_by_.empty()) return false; pulled_all_input_ = true; aggregation_it_ = aggregation_.begin(); @@ -3824,12 +3824,6 @@ UniqueCursorPtr Aggregate::MakeCursor(utils::MemoryResource *mem) const { return MakeUniqueCursorPtr(mem, *this, mem); } -auto Aggregate::AreAllAggregationsForCollecting() const -> bool { - return std::all_of(aggregations_.begin(), aggregations_.end(), [](const auto &agg) { - return agg.op == Aggregation::Op::COLLECT_LIST || agg.op == Aggregation::Op::COLLECT_MAP; - }); -} - Skip::Skip(const std::shared_ptr &input, Expression *expression) : input_(input), expression_(expression) {} diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index ba844796a..7bb971752 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -1759,8 +1759,6 @@ class Aggregate : public memgraph::query::plan::LogicalOperator { Aggregate(const std::shared_ptr &input, const std::vector &aggregations, const std::vector &group_by, const std::vector &remember); - auto AreAllAggregationsForCollecting() const -> bool; - bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; diff --git a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature index cff138432..80b0ca69a 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/aggregations.feature @@ -425,6 +425,4 @@ Feature: Aggregations """ MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id """ - Then the result should be: - | id | - | null | + Then the result should be empty diff --git a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature index cff138432..80b0ca69a 100644 --- a/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature +++ b/tests/gql_behave/tests/memgraph_V1_on_disk/features/aggregations.feature @@ -425,6 +425,4 @@ Feature: Aggregations """ MATCH (subnet:Subnet) WHERE FALSE WITH subnet, count(subnet.ip) as ips RETURN id(subnet) as id """ - Then the result should be: - | id | - | null | + Then the result should be empty diff --git a/tests/unit/query_plan_accumulate_aggregate.cpp b/tests/unit/query_plan_accumulate_aggregate.cpp index bbf3e0311..b1e9a62d0 100644 --- a/tests/unit/query_plan_accumulate_aggregate.cpp +++ b/tests/unit/query_plan_accumulate_aggregate.cpp @@ -250,30 +250,23 @@ TYPED_TEST(QueryPlanAggregateOps, WithData) { TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithGroupBy) { { auto results = this->AggregationResults(true, false, {Aggregation::Op::COUNT}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::SUM}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::AVG}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::MIN}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::MAX}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, false, {Aggregation::Op::COLLECT_LIST}); @@ -666,30 +659,23 @@ TYPED_TEST(QueryPlanAggregateOps, WithDataDistinct) { TYPED_TEST(QueryPlanAggregateOps, WithoutDataWithDistinctAndWithGroupBy) { { auto results = this->AggregationResults(true, true, {Aggregation::Op::COUNT}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::SUM}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Int); - EXPECT_EQ(results[0][0].ValueInt(), 0); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::AVG}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::MIN}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::MAX}); - EXPECT_EQ(results.size(), 1); - EXPECT_EQ(results[0][0].type(), TypedValue::Type::Null); + EXPECT_EQ(results.size(), 0); } { auto results = this->AggregationResults(true, true, {Aggregation::Op::COLLECT_LIST}); From 1d90b60f56b8acb1ac5a8003da362bd75caa1ed4 Mon Sep 17 00:00:00 2001 From: Andi Date: Tue, 21 Nov 2023 09:19:50 +0100 Subject: [PATCH 07/10] Add schema.assert (#1485) --- include/_mgp.hpp | 48 +++ include/mg_procedure.h | 59 +++ include/mgp.hpp | 82 +++- query_modules/schema.cpp | 465 +++++++++++++++++++++- src/query/db_accessor.cpp | 2 + src/query/db_accessor.hpp | 2 + src/query/interpreter.cpp | 5 +- src/query/procedure/mg_procedure_impl.cpp | 327 +++++++++++++++ src/storage/v2/inmemory/storage.cpp | 16 +- tests/e2e/query_modules/schema_test.py | 404 +++++++++++++++++++ tests/e2e/query_modules/workloads.yaml | 2 +- 11 files changed, 1373 insertions(+), 39 deletions(-) diff --git a/include/_mgp.hpp b/include/_mgp.hpp index fd286b6c6..58685b440 100644 --- a/include/_mgp.hpp +++ b/include/_mgp.hpp @@ -236,6 +236,54 @@ inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke(mgp // mgp_graph +inline bool create_label_index(mgp_graph *graph, const char *label) { + return MgInvoke(mgp_create_label_index, graph, label); +} + +inline bool drop_label_index(mgp_graph *graph, const char *label) { + return MgInvoke(mgp_drop_label_index, graph, label); +} + +inline mgp_list *list_all_label_indices(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_label_indices, graph, memory); +} + +inline bool create_label_property_index(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_create_label_property_index, graph, label, property); +} + +inline bool drop_label_property_index(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_drop_label_property_index, graph, label, property); +} + +inline mgp_list *list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_label_property_indices, graph, memory); +} + +inline bool create_existence_constraint(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_create_existence_constraint, graph, label, property); +} + +inline bool drop_existence_constraint(mgp_graph *graph, const char *label, const char *property) { + return MgInvoke(mgp_drop_existence_constraint, graph, label, property); +} + +inline mgp_list *list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_existence_constraints, graph, memory); +} + +inline bool create_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) { + return MgInvoke(mgp_create_unique_constraint, memgraph_graph, label, properties); +} + +inline bool drop_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) { + return MgInvoke(mgp_drop_unique_constraint, memgraph_graph, label, properties); +} + +inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory) { + return MgInvoke(mgp_list_all_unique_constraints, graph, memory); +} + inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke(mgp_graph_is_mutable, graph); } inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) { diff --git a/include/mg_procedure.h b/include/mg_procedure.h index 0bd831174..857c5f4dd 100644 --- a/include/mg_procedure.h +++ b/include/mg_procedure.h @@ -876,6 +876,65 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory, struct mgp_vertex **result); +/// Creates label index for given label. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if label index already exists, result will be 0, otherwise 1. +enum mgp_error mgp_create_label_index(struct mgp_graph *graph, const char *label, int *result); + +/// Drop label index. +enum mgp_error mgp_drop_label_index(struct mgp_graph *graph, const char *label, int *result); + +/// List all label indices. +enum mgp_error mgp_list_all_label_indices(struct mgp_graph *graph, struct mgp_memory *memory, struct mgp_list **result); + +/// Creates label-property index for given label and propery. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if label property index already exists, result will be 0, otherwise 1. +enum mgp_error mgp_create_label_property_index(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// Drops label-property index for given label and propery. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping label property index failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_label_property_index(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// List all label+property indices. +enum mgp_error mgp_list_all_label_property_indices(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + +/// Creates existence constraint for given label and property. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if creating existence constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_create_existence_constraint(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// Drops existence constraint for given label and property. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping existence constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_existence_constraint(struct mgp_graph *graph, const char *label, const char *property, + int *result); + +/// List all existence constraints. +enum mgp_error mgp_list_all_existence_constraints(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + +/// Creates unique constraint for given label and properties. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if creating unique constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_create_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties, + int *result); + +/// Drops unique constraint for given label and properties. +/// mgp_error::MGP_ERROR_NO_ERROR is always returned. +/// if dropping unique constraint failed, result will be 0, otherwise 1. +enum mgp_error mgp_drop_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties, + int *result); + +/// List all unique constraints +enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct mgp_memory *memory, + struct mgp_list **result); + /// Result is non-zero if the graph can be modified. /// If a graph is immutable, then vertices cannot be created or deleted, and all of the returned vertices will be /// immutable also. The same applies for edges. diff --git a/include/mgp.hpp b/include/mgp.hpp index bea3545bb..25e365e53 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -21,12 +21,10 @@ #include #include #include -#include -#include - -#include #include +#include #include +#include #include "_mgp.hpp" #include "mg_exceptions.hpp" @@ -1289,7 +1287,7 @@ class Value { std::string_view ValueString() const; std::string_view ValueString(); /// @pre Value type needs to be Type::List. - const List ValueList() const; + List ValueList() const; List ValueList(); /// @pre Value type needs to be Type::Map. const Map ValueMap() const; @@ -3651,7 +3649,7 @@ inline std::string_view Value::ValueString() { return mgp::value_get_string(ptr_); } -inline const List Value::ValueList() const { +inline List Value::ValueList() const { if (Type() != Type::List) { throw ValueException("Type of value is wrong: expected List."); } @@ -4279,9 +4277,77 @@ inline void AddParamsReturnsToProc(mgp_proc *proc, std::vector ¶m } } // namespace detail +inline bool CreateLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) { + return create_label_index(memgaph_graph, label.data()); +} + +inline bool DropLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) { + return drop_label_index(memgaph_graph, label.data()); +} + +inline List ListAllLabelIndices(mgp_graph *memgraph_graph) { + auto *label_indices = mgp::MemHandlerCallback(list_all_label_indices, memgraph_graph); + if (label_indices == nullptr) { + throw ValueException("Couldn't list all label indices"); + } + return List(label_indices); +} + +inline bool CreateLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label, + const std::string_view property) { + return create_label_property_index(memgaph_graph, label.data(), property.data()); +} + +inline bool DropLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label, + const std::string_view property) { + return drop_label_property_index(memgaph_graph, label.data(), property.data()); +} + +inline List ListAllLabelPropertyIndices(mgp_graph *memgraph_graph) { + auto *label_property_indices = mgp::MemHandlerCallback(list_all_label_property_indices, memgraph_graph); + if (label_property_indices == nullptr) { + throw ValueException("Couldn't list all label+property indices"); + } + return List(label_property_indices); +} + +inline bool CreateExistenceConstraint(mgp_graph *memgraph_graph, const std::string_view label, + const std::string_view property) { + return create_existence_constraint(memgraph_graph, label.data(), property.data()); +} + +inline bool DropExistenceConstraint(mgp_graph *memgraph_graph, const std::string_view label, + const std::string_view property) { + return drop_existence_constraint(memgraph_graph, label.data(), property.data()); +} + +inline List ListAllExistenceConstraints(mgp_graph *memgraph_graph) { + auto *existence_constraints = mgp::MemHandlerCallback(list_all_existence_constraints, memgraph_graph); + if (existence_constraints == nullptr) { + throw ValueException("Couldn't list all existence_constraints"); + } + return List(existence_constraints); +} + +inline bool CreateUniqueConstraint(mgp_graph *memgraph_graph, const std::string_view label, mgp_value *properties) { + return create_unique_constraint(memgraph_graph, label.data(), properties); +} + +inline bool DropUniqueConstraint(mgp_graph *memgraph_graph, const std::string_view label, mgp_value *properties) { + return drop_unique_constraint(memgraph_graph, label.data(), properties); +} + +inline List ListAllUniqueConstraints(mgp_graph *memgraph_graph) { + auto *unique_constraints = mgp::MemHandlerCallback(list_all_unique_constraints, memgraph_graph); + if (unique_constraints == nullptr) { + throw ValueException("Couldn't list all unique_constraints"); + } + return List(unique_constraints); +} + void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType proc_type, std::vector parameters, std::vector returns, mgp_module *module, - mgp_memory *memory) { + mgp_memory * /*memory*/) { auto *proc = (proc_type == ProcedureType::Read) ? mgp::module_add_read_procedure(module, name.data(), callback) : mgp::module_add_write_procedure(module, name.data(), callback); detail::AddParamsReturnsToProc(proc, parameters, returns); @@ -4289,7 +4355,7 @@ void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType pro void AddBatchProcedure(mgp_proc_cb callback, mgp_proc_initializer initializer, mgp_proc_cleanup cleanup, std::string_view name, ProcedureType proc_type, std::vector parameters, - std::vector returns, mgp_module *module, mgp_memory *memory) { + std::vector returns, mgp_module *module, mgp_memory * /*memory*/) { auto *proc = (proc_type == ProcedureType::Read) ? mgp::module_add_batch_read_procedure(module, name.data(), callback, initializer, cleanup) : mgp::module_add_batch_write_procedure(module, name.data(), callback, initializer, cleanup); diff --git a/query_modules/schema.cpp b/query_modules/schema.cpp index 436e00716..d5a657e98 100644 --- a/query_modules/schema.cpp +++ b/query_modules/schema.cpp @@ -10,18 +10,33 @@ // licenses/APL.txt. #include +#include "utils/string.hpp" + +#include namespace Schema { -/*NodeTypeProperties and RelTypeProperties constants*/ +constexpr std::string_view kStatusKept = "Kept"; +constexpr std::string_view kStatusCreated = "Created"; +constexpr std::string_view kStatusDropped = "Dropped"; constexpr std::string_view kReturnNodeType = "nodeType"; constexpr std::string_view kProcedureNodeType = "node_type_properties"; constexpr std::string_view kProcedureRelType = "rel_type_properties"; +constexpr std::string_view kProcedureAssert = "assert"; constexpr std::string_view kReturnLabels = "nodeLabels"; constexpr std::string_view kReturnRelType = "relType"; constexpr std::string_view kReturnPropertyName = "propertyName"; constexpr std::string_view kReturnPropertyType = "propertyTypes"; constexpr std::string_view kReturnMandatory = "mandatory"; +constexpr std::string_view kReturnLabel = "label"; +constexpr std::string_view kReturnKey = "key"; +constexpr std::string_view kReturnKeys = "keys"; +constexpr std::string_view kReturnUnique = "unique"; +constexpr std::string_view kReturnAction = "action"; +constexpr std::string_view kParameterIndices = "indices"; +constexpr std::string_view kParameterUniqueConstraints = "unique_constraints"; +constexpr std::string_view kParameterExistenceConstraints = "existence_constraints"; +constexpr std::string_view kParameterDropExisting = "drop_existing"; std::string TypeOf(const mgp::Type &type); @@ -35,6 +50,7 @@ void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, con void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); +void Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory); } // namespace Schema /*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this @@ -92,21 +108,22 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t record.Insert(std::string(kReturnMandatory).c_str(), mandatory); } -void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { +void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, + mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); for (auto node : graph.Nodes()) { - std::string type = ""; + std::string type; mgp::List labels = mgp::List(); for (auto label : node.Labels()) { labels.AppendExtend(mgp::Value(label)); type += ":`" + std::string(label) + "`"; } - if (node.Properties().size() == 0) { + if (node.Properties().empty()) { auto record = record_factory.NewRecord(); ProcessPropertiesNode(record, type, labels, "", "", false); continue; @@ -126,16 +143,15 @@ void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_r } } -void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { +void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; - ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); for (auto rel : graph.Relationships()) { std::string type = ":`" + std::string(rel.Type()) + "`"; - if (rel.Properties().size() == 0) { + if (rel.Properties().empty()) { auto record = record_factory.NewRecord(); ProcessPropertiesRel(record, type, "", "", false); continue; @@ -155,29 +171,436 @@ void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_re } } +void InsertRecordForLabelIndex(const auto &record_factory, const std::string_view label, + const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), ""); + record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List()); + record.Insert(std::string(Schema::kReturnUnique).c_str(), false); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void InsertRecordForUniqueConstraint(const auto &record_factory, const std::string_view label, + const mgp::List &properties, const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), properties.ToString()); + record.Insert(std::string(Schema::kReturnKeys).c_str(), properties); + record.Insert(std::string(Schema::kReturnUnique).c_str(), true); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void InsertRecordForLabelPropertyIndexAndExistenceConstraint(const auto &record_factory, const std::string_view label, + const std::string_view property, + const std::string_view status) { + auto record = record_factory.NewRecord(); + record.Insert(std::string(Schema::kReturnLabel).c_str(), label); + record.Insert(std::string(Schema::kReturnKey).c_str(), property); + record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List({mgp::Value(property)})); + record.Insert(std::string(Schema::kReturnUnique).c_str(), false); + record.Insert(std::string(Schema::kReturnAction).c_str(), status); +} + +void ProcessCreatingLabelIndex(const std::string_view label, const std::set &existing_label_indices, + mgp_graph *memgraph_graph, const auto &record_factory) { + if (existing_label_indices.contains(label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusKept); + } else if (mgp::CreateLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated); + } +} + +template +void ProcessCreatingLabelPropertyIndexAndExistenceConstraint(const std::string_view label, + const std::string_view property, + const std::set &existing_collection, + const TFunc &func_creation, mgp_graph *memgraph_graph, + const auto &record_factory) { + const auto label_property_search_key = std::string(label) + ":" + std::string(property); + if (existing_collection.contains(label_property_search_key)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusKept); + } else if (func_creation(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusCreated); + } +} + +/// We collect properties for which index was created. +using AssertedIndices = std::set>; +AssertedIndices CreateIndicesForLabel(const std::string_view label, const mgp::Value &properties_val, + mgp_graph *memgraph_graph, const auto &record_factory, + const std::set &existing_label_indices, + const std::set &existing_label_property_indices) { + AssertedIndices asserted_indices; + if (!properties_val.IsList()) { + return {}; + } + if (const auto properties = properties_val.ValueList(); + properties.Empty() && mgp::CreateLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated); + asserted_indices.emplace(""); + } else { + std::for_each(properties.begin(), properties.end(), + [&label, &existing_label_indices, &existing_label_property_indices, &memgraph_graph, &record_factory, + &asserted_indices](const mgp::Value &property) { + if (!property.IsString()) { + return; + } + const auto property_str = property.ValueString(); + if (property_str.empty()) { + ProcessCreatingLabelIndex(label, existing_label_indices, memgraph_graph, record_factory); + asserted_indices.emplace(""); + } else { + ProcessCreatingLabelPropertyIndexAndExistenceConstraint( + label, property_str, existing_label_property_indices, mgp::CreateLabelPropertyIndex, + memgraph_graph, record_factory); + asserted_indices.emplace(property_str); + } + }); + } + return asserted_indices; +} + +void ProcessIndices(const mgp::Map &indices_map, mgp_graph *memgraph_graph, const auto &record_factory, + bool drop_existing) { + auto mgp_existing_label_indices = mgp::ListAllLabelIndices(memgraph_graph); + auto mgp_existing_label_property_indices = mgp::ListAllLabelPropertyIndices(memgraph_graph); + + std::set existing_label_indices; + std::transform(mgp_existing_label_indices.begin(), mgp_existing_label_indices.end(), + std::inserter(existing_label_indices, existing_label_indices.begin()), + [](const mgp::Value &index) { return index.ValueString(); }); + + std::set existing_label_property_indices; + std::transform(mgp_existing_label_property_indices.begin(), mgp_existing_label_property_indices.end(), + std::inserter(existing_label_property_indices, existing_label_property_indices.begin()), + [](const mgp::Value &index) { return index.ValueString(); }); + + std::set asserted_label_indices; + std::set asserted_label_property_indices; + + auto merge_label_property = [](const std::string &label, const std::string &property) { + return label + ":" + property; + }; + + for (const auto &index : indices_map) { + const std::string_view label = index.key; + const mgp::Value &properties_val = index.value; + + AssertedIndices asserted_indices_new = CreateIndicesForLabel( + label, properties_val, memgraph_graph, record_factory, existing_label_indices, existing_label_property_indices); + + if (!drop_existing) { + continue; + } + std::ranges::for_each(asserted_indices_new, [&asserted_label_indices, &asserted_label_property_indices, label, + &merge_label_property](const std::string &property) { + if (property.empty()) { + asserted_label_indices.emplace(label); + } else { + asserted_label_property_indices.emplace(merge_label_property(std::string(label), property)); + } + }); + } + + if (!drop_existing) { + return; + } + + std::set label_indices_to_drop; + std::ranges::set_difference(existing_label_indices, asserted_label_indices, + std::inserter(label_indices_to_drop, label_indices_to_drop.begin())); + + std::ranges::for_each(label_indices_to_drop, [memgraph_graph, &record_factory](const std::string_view label) { + if (mgp::DropLabelIndex(memgraph_graph, label)) { + InsertRecordForLabelIndex(record_factory, label, Schema::kStatusDropped); + } + }); + + std::set label_property_indices_to_drop; + std::ranges::set_difference(existing_label_property_indices, asserted_label_property_indices, + std::inserter(label_property_indices_to_drop, label_property_indices_to_drop.begin())); + + auto decouple_label_property = [](const std::string_view label_property) { + const auto label_size = label_property.find(':'); + const auto label = std::string(label_property.substr(0, label_size)); + const auto property = std::string(label_property.substr(label_size + 1)); + return std::make_pair(label, property); + }; + + std::ranges::for_each(label_property_indices_to_drop, [memgraph_graph, &record_factory, decouple_label_property]( + const std::string_view label_property) { + const auto [label, property] = decouple_label_property(label_property); + if (mgp::DropLabelPropertyIndex(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped); + } + }); +} + +using ExistenceConstraintsStorage = std::set; + +ExistenceConstraintsStorage CreateExistenceConstraintsForLabel( + const std::string_view label, const mgp::Value &properties_val, mgp_graph *memgraph_graph, + const auto &record_factory, const std::set &existing_existence_constraints) { + ExistenceConstraintsStorage asserted_existence_constraints; + if (!properties_val.IsList()) { + return asserted_existence_constraints; + } + + auto validate_property = [](const mgp::Value &property) -> bool { + return property.IsString() && !property.ValueString().empty(); + }; + + const auto &properties = properties_val.ValueList(); + std::for_each(properties.begin(), properties.end(), + [&label, &existing_existence_constraints, &asserted_existence_constraints, &memgraph_graph, + &record_factory, &validate_property](const mgp::Value &property) { + if (!validate_property(property)) { + return; + } + const std::string_view property_str = property.ValueString(); + asserted_existence_constraints.emplace(property_str); + ProcessCreatingLabelPropertyIndexAndExistenceConstraint( + label, property_str, existing_existence_constraints, mgp::CreateExistenceConstraint, + memgraph_graph, record_factory); + }); + return asserted_existence_constraints; +} + +void ProcessExistenceConstraints(const mgp::Map &existence_constraints_map, mgp_graph *memgraph_graph, + const auto &record_factory, bool drop_existing) { + auto mgp_existing_existence_constraints = mgp::ListAllExistenceConstraints(memgraph_graph); + std::set existing_existence_constraints; + std::transform(mgp_existing_existence_constraints.begin(), mgp_existing_existence_constraints.end(), + std::inserter(existing_existence_constraints, existing_existence_constraints.begin()), + [](const mgp::Value &constraint) { return constraint.ValueString(); }); + + auto merge_label_property = [](const std::string_view label, const std::string_view property) { + auto str = std::string(label) + ":"; + str += property; + return str; + }; + + ExistenceConstraintsStorage asserted_existence_constraints; + + for (const auto &existing_constraint : existence_constraints_map) { + const std::string_view label = existing_constraint.key; + const mgp::Value &properties_val = existing_constraint.value; + auto asserted_existence_constraints_new = CreateExistenceConstraintsForLabel( + label, properties_val, memgraph_graph, record_factory, existing_existence_constraints); + if (!drop_existing) { + continue; + } + + std::ranges::for_each(asserted_existence_constraints_new, [&asserted_existence_constraints, &merge_label_property, + label](const std::string_view property) { + asserted_existence_constraints.emplace(merge_label_property(label, property)); + }); + } + + if (!drop_existing) { + return; + } + + std::set existence_constraints_to_drop; + std::ranges::set_difference(existing_existence_constraints, asserted_existence_constraints, + std::inserter(existence_constraints_to_drop, existence_constraints_to_drop.begin())); + + auto decouple_label_property = [](const std::string_view label_property) { + const auto label_size = label_property.find(':'); + const auto label = std::string(label_property.substr(0, label_size)); + const auto property = std::string(label_property.substr(label_size + 1)); + return std::make_pair(label, property); + }; + + std::ranges::for_each(existence_constraints_to_drop, [&](const std::string_view label_property) { + const auto [label, property] = decouple_label_property(label_property); + if (mgp::DropExistenceConstraint(memgraph_graph, label, property)) { + InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped); + } + }); +} + +using AssertedUniqueConstraintsStorage = std::set>; +AssertedUniqueConstraintsStorage CreateUniqueConstraintsForLabel( + const std::string_view label, const mgp::Value &unique_props_nested, + const std::map &existing_unique_constraints, + mgp_graph *memgraph_graph, const auto &record_factory) { + AssertedUniqueConstraintsStorage asserted_unique_constraints; + if (!unique_props_nested.IsList()) { + return asserted_unique_constraints; + } + + auto validate_unique_constraint_props = [](const mgp::Value &properties) -> bool { + if (!properties.IsList()) { + return false; + } + const auto &properties_list = properties.ValueList(); + if (properties_list.Empty()) { + return false; + } + return std::all_of(properties_list.begin(), properties_list.end(), [](const mgp::Value &property) { + return property.IsString() && !property.ValueString().empty(); + }); + }; + + auto unique_constraint_exists = + [](const std::string_view label, const std::set &properties, + const std::map &existing_unique_constraints) -> bool { + auto iter = existing_unique_constraints.find(label); + if (iter == existing_unique_constraints.end()) { + return false; + } + return iter->second.find(properties) != iter->second.end(); + }; + + for (const auto unique_props_nested_list = unique_props_nested.ValueList(); + const auto &properties : unique_props_nested_list) { + if (!validate_unique_constraint_props(properties)) { + continue; + } + const auto properties_list = properties.ValueList(); + std::set properties_coll; + std::transform(properties_list.begin(), properties_list.end(), + std::inserter(properties_coll, properties_coll.begin()), + [](const mgp::Value &property) { return property.ValueString(); }); + + if (unique_constraint_exists(label, properties_coll, existing_unique_constraints)) { + InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusKept); + } else if (mgp::CreateUniqueConstraint(memgraph_graph, label, properties.ptr())) { + InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusCreated); + } + asserted_unique_constraints.emplace(std::move(properties_coll)); + } + return asserted_unique_constraints; +} + +void ProcessUniqueConstraints(const mgp::Map &unique_constraints_map, mgp_graph *memgraph_graph, + const auto &record_factory, bool drop_existing) { + auto mgp_existing_unique_constraints = mgp::ListAllUniqueConstraints(memgraph_graph); + // label-unique_constraints pair + std::map existing_unique_constraints; + for (const auto &constraint : mgp_existing_unique_constraints) { + auto constraint_list = constraint.ValueList(); + std::set properties; + for (int i = 1; i < constraint_list.Size(); i++) { + properties.emplace(constraint_list[i].ValueString()); + } + const std::string_view label = constraint_list[0].ValueString(); + auto [it, inserted] = existing_unique_constraints.try_emplace(label, AssertedUniqueConstraintsStorage{properties}); + if (!inserted) { + it->second.emplace(std::move(properties)); + } + } + + std::map asserted_unique_constraints; + + for (const auto &[label, unique_props_nested] : unique_constraints_map) { + auto asserted_unique_constraints_new = CreateUniqueConstraintsForLabel( + label, unique_props_nested, existing_unique_constraints, memgraph_graph, record_factory); + if (drop_existing) { + asserted_unique_constraints.emplace(label, std::move(asserted_unique_constraints_new)); + } + } + + if (!drop_existing) { + return; + } + + std::vector>> unique_constraints_to_drop; + + // Check for each label for we found existing constraint in the DB whether it was asserted. + // If no unique constraint was found with label, we can drop all unique constraints for this label. (if branch) + // If some unique constraint was found with label, we can drop only those unique constraints that were not asserted. + // (else branch.) + std::ranges::for_each(existing_unique_constraints, [&asserted_unique_constraints, &unique_constraints_to_drop]( + const auto &existing_label_unique_constraints) { + const auto &label = existing_label_unique_constraints.first; + const auto &existing_unique_constraints_for_label = existing_label_unique_constraints.second; + const auto &asserted_unique_constraints_for_label = asserted_unique_constraints.find(label); + if (asserted_unique_constraints_for_label == asserted_unique_constraints.end()) { + std::ranges::for_each( + std::make_move_iterator(existing_unique_constraints_for_label.begin()), + std::make_move_iterator(existing_unique_constraints_for_label.end()), + [&unique_constraints_to_drop, &label](std::set existing_unique_constraint_for_label) { + unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label)); + }); + } else { + const auto &asserted_unique_constraints_for_label_coll = asserted_unique_constraints_for_label->second; + std::ranges::for_each( + std::make_move_iterator(existing_unique_constraints_for_label.begin()), + std::make_move_iterator(existing_unique_constraints_for_label.end()), + [&unique_constraints_to_drop, &label, &asserted_unique_constraints_for_label_coll]( + std::set existing_unique_constraint_for_label) { + if (!asserted_unique_constraints_for_label_coll.contains(existing_unique_constraint_for_label)) { + unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label)); + } + }); + } + }); + std::ranges::for_each( + unique_constraints_to_drop, [memgraph_graph, &record_factory](const auto &label_unique_constraint) { + const auto &[label, unique_constraint] = label_unique_constraint; + + auto unique_constraint_list = mgp::List(); + std::ranges::for_each(unique_constraint, [&unique_constraint_list](const std::string_view &property) { + unique_constraint_list.AppendExtend(mgp::Value(property)); + }); + + if (mgp::DropUniqueConstraint(memgraph_graph, label, mgp::Value(unique_constraint_list).ptr())) { + InsertRecordForUniqueConstraint(record_factory, label, unique_constraint_list, Schema::kStatusDropped); + } + }); +} + +void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { + mgp::MemoryDispatcherGuard guard{memory}; + const auto record_factory = mgp::RecordFactory(result); + auto arguments = mgp::List(args); + auto indices_map = arguments[0].ValueMap(); + auto unique_constraints_map = arguments[1].ValueMap(); + auto existence_constraints_map = arguments[2].ValueMap(); + auto drop_existing = arguments[3].ValueBool(); + + ProcessIndices(indices_map, memgraph_graph, record_factory, drop_existing); + ProcessExistenceConstraints(existence_constraints_map, memgraph_graph, record_factory, drop_existing); + ProcessUniqueConstraints(unique_constraints_map, memgraph_graph, record_factory, drop_existing); +} + extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { try { mgp::MemoryDispatcherGuard guard{memory}; ; - AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read, - {}, - {mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}), - mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any), - mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)}, + AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {}, + {mgp::Return(Schema::kReturnNodeType, mgp::Type::String), + mgp::Return(Schema::kReturnLabels, {mgp::Type::List, mgp::Type::String}), + mgp::Return(Schema::kReturnPropertyName, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any), + mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)}, module, memory); - AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read, - {}, - {mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String), - mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any), - mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)}, + AddProcedure(Schema::RelTypeProperties, Schema::kProcedureRelType, mgp::ProcedureType::Read, {}, + {mgp::Return(Schema::kReturnRelType, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyName, mgp::Type::String), + mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any), + mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)}, module, memory); - + AddProcedure( + Schema::Assert, Schema::kProcedureAssert, mgp::ProcedureType::Read, + { + mgp::Parameter(Schema::kParameterIndices, {mgp::Type::Map, mgp::Type::Any}), + mgp::Parameter(Schema::kParameterUniqueConstraints, {mgp::Type::Map, mgp::Type::Any}), + mgp::Parameter(Schema::kParameterExistenceConstraints, {mgp::Type::Map, mgp::Type::Any}, + mgp::Value(mgp::Map{})), + mgp::Parameter(Schema::kParameterDropExisting, mgp::Type::Bool, mgp::Value(true)), + }, + {mgp::Return(Schema::kReturnLabel, mgp::Type::String), mgp::Return(Schema::kReturnKey, mgp::Type::String), + mgp::Return(Schema::kReturnKeys, {mgp::Type::List, mgp::Type::String}), + mgp::Return(Schema::kReturnUnique, mgp::Type::Bool), mgp::Return(Schema::kReturnAction, mgp::Type::String)}, + module, memory); } catch (const std::exception &e) { + std::cerr << "Error while initializing query module: " << e.what() << std::endl; return 1; } diff --git a/src/query/db_accessor.cpp b/src/query/db_accessor.cpp index 0250ab695..df3fb808a 100644 --- a/src/query/db_accessor.cpp +++ b/src/query/db_accessor.cpp @@ -139,6 +139,8 @@ std::optional SubgraphDbAccessor::FindVertex(storage::Gid gid, s query::Graph *SubgraphDbAccessor::getGraph() { return graph_; } +DbAccessor *SubgraphDbAccessor::GetAccessor() { return &db_accessor_; } + VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; } storage::Result SubgraphVertexAccessor::OutEdges(storage::View view) const { diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index d6114edaf..75ec1e9ae 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -694,6 +694,8 @@ class SubgraphDbAccessor final { std::optional FindVertex(storage::Gid gid, storage::View view); Graph *getGraph(); + + DbAccessor *GetAccessor(); }; } // namespace memgraph::query diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 30bb4eca2..5aad0ff07 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -147,6 +147,8 @@ void memgraph::query::CurrentDB::CleanupDBTransaction(bool abort) { namespace memgraph::query { +constexpr std::string_view kSchemaAssert = "SCHEMA.ASSERT"; + template constexpr auto kAlwaysFalse = false; @@ -3715,7 +3717,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, // TODO: ATM only a single database, will change when we have multiple database transactions bool could_commit = utils::Downcast(parsed_query.query) != nullptr; bool unique = utils::Downcast(parsed_query.query) != nullptr || - utils::Downcast(parsed_query.query) != nullptr; + utils::Downcast(parsed_query.query) != nullptr || + upper_case_query.find(kSchemaAssert) != std::string::npos; SetupDatabaseTransaction(could_commit, unique); } diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index 2a176f2ed..f87377ba5 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -2535,6 +2535,333 @@ mgp_error mgp_graph_get_vertex_by_id(mgp_graph *graph, mgp_vertex_id id, mgp_mem result); } +mgp_error mgp_create_label_index(mgp_graph *graph, const char *label, int *result) { + return WrapExceptions( + [graph, label]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [label_id](memgraph::query::DbAccessor *impl) { return impl->CreateIndex(label_id); }, + [label_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateIndex(label_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_label_index(mgp_graph *graph, const char *label, int *result) { + return WrapExceptions( + [graph, label]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [label_id](memgraph::query::DbAccessor *impl) { return impl->DropIndex(label_id); }, + [label_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropIndex(label_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_list_all_label_indices(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto index_res = std::visit( + memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllIndices().label; }, + [](memgraph::query::SubgraphDbAccessor *impl) { return impl->GetAccessor()->ListAllIndices().label; }}, + graph->impl); + if (const auto err = mgp_list_make_empty(index_res.size(), memory, result); err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of creating list"); + } + for (const auto &label : index_res) { + const auto label_id_str = std::visit([label](const auto *impl) { return impl->LabelToName(label); }, graph->impl); + + mgp_value *label_value = nullptr; + if (const auto err_str = mgp_value_make_string(label_id_str.c_str(), memory, &label_value); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of creating label value"); + } + if (const auto err_list = mgp_list_append_extend(*result, label_value); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label indices failed due to failure of appending label value"); + } + mgp_value_destroy(label_value); + } + }); +} + +mgp_error mgp_create_label_property_index(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->CreateIndex(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateIndex(label_id, property_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_label_property_index(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto index_res = + std::visit(memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->DropIndex(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropIndex(label_id, property_id); + }}, + graph->impl); + return index_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error create_and_append_label_property_to_mgp_list(mgp_graph *graph, mgp_memory *memory, mgp_list **result, + const auto &label_property_pair) { + return WrapExceptions([graph, memory, result, &label_property_pair]() { + const auto label_id_str = std::visit( + [label_id = label_property_pair.first](const auto *impl) { return impl->LabelToName(label_id); }, graph->impl); + const auto property_id_str = std::visit( + [property_id = label_property_pair.second](const auto *impl) { return impl->PropertyToName(property_id); }, + graph->impl); + + // This is hack to avoid dealing with pairs + mgp_value *label_property = nullptr; + auto final_str = label_id_str + ":"; + final_str += property_id_str; + + if (const auto err_str = mgp_value_make_string(final_str.c_str(), memory, &label_property); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Creating a list of label+property pairs failed due to failure of creating label+property value"); + } + if (const auto err_list = mgp_list_append_extend(*result, label_property); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Creating a list of label-property pairs due to failure of appending label+property value"); + } + + mgp_value_destroy(label_property); + }); +} + +mgp_error mgp_list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto index_res = + std::visit(memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllIndices().label_property; }, + [](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->ListAllIndices().label_property; + }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(index_res.size(), memory, result); err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all label+property indices failed due to failure of creating list"); + } + + for (const auto &label_property_pair : index_res) { + if (const auto err = create_and_append_label_property_to_mgp_list(graph, memory, result, label_property_pair); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Listing all label+property indices failed due to failure of appending label+property value"); + } + } + }); +} + +mgp_error mgp_create_existence_constraint(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto exist_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->CreateExistenceConstraint(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateExistenceConstraint(label_id, property_id); + }}, + graph->impl); + return exist_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_existence_constraint(mgp_graph *graph, const char *label, const char *property, int *result) { + return WrapExceptions( + [graph, label, property]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + const auto property_id = + std::visit([property](auto *impl) { return impl->NameToProperty(property); }, graph->impl); + const auto exist_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_id](memgraph::query::DbAccessor *impl) { + return impl->DropExistenceConstraint(label_id, property_id); + }, + [label_id, property_id](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropExistenceConstraint(label_id, property_id); + }}, + graph->impl); + return exist_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto constraint_res = + std::visit(memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllConstraints().existence; }, + [](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->ListAllConstraints().existence; + }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(constraint_res.size(), memory, result); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all existence constraints failed due to failure of creating a list"); + } + + for (const auto &label_property_pair : constraint_res) { + if (const auto err = create_and_append_label_property_to_mgp_list(graph, memory, result, label_property_pair); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error( + "Listing all existence constraints failed due to failure of appending label+property value"); + } + } + }); +} + +mgp_error mgp_create_unique_constraint(mgp_graph *graph, const char *label, mgp_value *properties, int *result) { + return WrapExceptions( + [graph, label, properties]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + std::set property_ids; + for (const auto &elem : properties->list_v->elems) { + property_ids.insert(std::visit( + [prop_str = elem.string_v](auto *impl) { return impl->NameToProperty(prop_str); }, graph->impl)); + } + + const auto unique_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_ids](memgraph::query::DbAccessor *impl) { + return impl->CreateUniqueConstraint(label_id, property_ids); + }, + [label_id, property_ids](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->CreateUniqueConstraint(label_id, property_ids); + }}, + graph->impl); + return unique_res.HasError() ? 0 : 1; + }, + result); +} + +mgp_error mgp_drop_unique_constraint(mgp_graph *graph, const char *label, mgp_value *properties, int *result) { + return WrapExceptions( + [graph, label, properties]() { + const auto label_id = std::visit([label](auto *impl) { return impl->NameToLabel(label); }, graph->impl); + std::set property_ids; + for (const auto &elem : properties->list_v->elems) { + property_ids.insert(std::visit( + [prop_str = elem.string_v](auto *impl) { return impl->NameToProperty(prop_str); }, graph->impl)); + } + + const auto unique_res = std::visit( + memgraph::utils::Overloaded{[label_id, property_ids](memgraph::query::DbAccessor *impl) { + return impl->DropUniqueConstraint(label_id, property_ids); + }, + [label_id, property_ids](memgraph::query::SubgraphDbAccessor *impl) { + return impl->GetAccessor()->DropUniqueConstraint(label_id, property_ids); + }}, + graph->impl); + return unique_res == memgraph::storage::UniqueConstraints::DeletionStatus::SUCCESS ? 1 : 0; + }, + result); +} + +mgp_error mgp_list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory, mgp_list **result) { + return WrapExceptions([graph, memory, result]() { + const auto constraints_res = std::visit( + memgraph::utils::Overloaded{ + [](memgraph::query::DbAccessor *impl) { return impl->ListAllConstraints().unique; }, + [](memgraph::query::SubgraphDbAccessor *impl) { return impl->GetAccessor()->ListAllConstraints().unique; }}, + graph->impl); + + if (const auto err = mgp_list_make_empty(constraints_res.size(), memory, result); + err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a list"); + } + + for (const auto &label_properties_pair : constraints_res) { + const std::string label_id_str = + std::visit([label_id = label_properties_pair.first](const auto *impl) { return impl->LabelToName(label_id); }, + graph->impl); + const std::vector properties_str = std::visit( + [property_ids = label_properties_pair.second](const auto *impl) { + std::vector property_ids_str; + property_ids_str.reserve(property_ids.size()); + std::transform(property_ids.begin(), property_ids.end(), std::back_inserter(property_ids_str), + [impl](const auto &property_id) { return impl->PropertyToName(property_id); }); + return property_ids_str; + }, + graph->impl); + + mgp_list *label_properties_mgp_list = nullptr; + if (const auto properties_mgp_list_err = + mgp_list_make_empty(properties_str.size() + 1, memory, &label_properties_mgp_list); + properties_mgp_list_err != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating an inner list"); + } + + mgp_value *mgp_value_label = nullptr; + if (const auto err_label = mgp_value_make_string(label_id_str.c_str(), memory, &mgp_value_label); + err_label != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a label value"); + } + if (const auto err_label_into_list = mgp_list_append_extend(label_properties_mgp_list, mgp_value_label); + err_label_into_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of appending a label value"); + } + + mgp_value_destroy(mgp_value_label); + + for (const std::string &property_str : properties_str) { + mgp_value *property_mgp_value = nullptr; + if (const auto err_str = mgp_value_make_string(property_str.c_str(), memory, &property_mgp_value); + err_str != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating a property value"); + } + if (const auto err_list = mgp_list_append_extend(label_properties_mgp_list, property_mgp_value); + err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of appending a property value"); + } + mgp_value_destroy(property_mgp_value); + } + mgp_value value(label_properties_mgp_list, label_properties_mgp_list->GetMemoryResource()); + + if (const auto err_list = mgp_list_append_extend(*result, &value); err_list != mgp_error::MGP_ERROR_NO_ERROR) { + throw std::logic_error("Listing all unique constraints failed due to failure of creating label+property value"); + } + mgp_value_destroy(&value); + } + }); +} + mgp_error mgp_graph_is_mutable(mgp_graph *graph, int *result) { *result = MgpGraphIsMutable(*graph) ? 1 : 0; return mgp_error::MGP_ERROR_NO_ERROR; diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index d0d1dd071..9f00081f6 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -1003,7 +1003,7 @@ void InMemoryStorage::InMemoryAccessor::FinalizeTransaction() { } utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex(LabelId label) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating label index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); if (!mem_label_index->CreateIndex(label, in_memory->vertices_.access(), std::nullopt)) { @@ -1017,7 +1017,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex( LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating label-property index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_property_index = static_cast(in_memory->indices_.label_property_index_.get()); @@ -1031,7 +1031,7 @@ utils::BasicResult InMemoryStorage::InMemoryA } utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex(LabelId label) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping label index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); if (!mem_label_index->DropIndex(label)) { @@ -1045,7 +1045,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex( LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping label-property index requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_label_property_index = static_cast(in_memory->indices_.label_property_index_.get()); @@ -1060,7 +1060,7 @@ utils::BasicResult InMemoryStorage::InMemoryA utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating existence requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); if (existence_constraints->ConstraintExists(label, property)) { @@ -1078,7 +1078,7 @@ InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, Prop utils::BasicResult InMemoryStorage::InMemoryAccessor::DropExistenceConstraint(LabelId label, PropertyId property) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping existence constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); if (!existence_constraints->DropConstraint(label, property)) { @@ -1090,7 +1090,7 @@ InMemoryStorage::InMemoryAccessor::DropExistenceConstraint(LabelId label, Proper utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const std::set &properties) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Creating unique constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_unique_constraints = static_cast(in_memory->constraints_.unique_constraints_.get()); @@ -1107,7 +1107,7 @@ InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const s UniqueConstraints::DeletionStatus InMemoryStorage::InMemoryAccessor::DropUniqueConstraint( LabelId label, const std::set &properties) { - MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + MG_ASSERT(unique_guard_.owns_lock(), "Dropping unique constraint requires a unique access to the storage!"); auto *in_memory = static_cast(storage_); auto *mem_unique_constraints = static_cast(in_memory->constraints_.unique_constraints_.get()); diff --git a/tests/e2e/query_modules/schema_test.py b/tests/e2e/query_modules/schema_test.py index e61d9df1b..515514a74 100644 --- a/tests/e2e/query_modules/schema_test.py +++ b/tests/e2e/query_modules/schema_test.py @@ -15,6 +15,410 @@ import pytest from common import connect, execute_and_fetch_all +def test_assert_creates_label_index_empty_list(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: []}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_creates_label_index_empty_string(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_index_wrong_properties_type(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ''}, {}) YIELD * RETURN *;", + ) + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + + +def test_assert_property_is_not_a_string(): + cursor = connect().cursor() + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['name', 1]}, {}) YIELD * RETURN *;", + ) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label+property", "Person", "name", 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + + +def test_assert_creates_label_index_multiple_empty_strings(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', '', '', '']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "", [], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + + +def test_assert_creates_label_property_index(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['name']}, {}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "name", ["name"], "Person", False)] + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label+property", "Person", "name", 0)] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + + +def test_assert_creates_multiple_indices(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id', 'name'], Ball: ['', 'size', 'size', '']}, {}) YIELD * RETURN *;", + ) + ) + assert len(results) == 5 + assert results[0] == ("Created", "", [], "Ball", False) + assert results[1] == ("Created", "size", ["size"], "Ball", False) + assert results[2] == ("Created", "", [], "Person", False) + assert results[3] == ("Created", "id", ["id"], "Person", False) + assert results[4] == ("Created", "name", ["name"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert len(show_index_results) == 5 + assert show_index_results[0] == ("label", "Ball", None, 0) + assert show_index_results[1] == ("label", "Person", None, 0) + assert show_index_results[2] == ("label+property", "Ball", "size", 0) + assert show_index_results[3] == ("label+property", "Person", "id", 0) + assert show_index_results[4] == ("label+property", "Person", "name", 0) + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(name);") + execute_and_fetch_all(cursor, "DROP INDEX ON :Ball;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Ball(size);") + + +def test_assert_creates_existence_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert results == [ + ("Created", "name", ["name"], "Person", False), + ("Created", "surname", ["surname"], "Person", False), + ] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name"), ("exists", "Person", "surname")] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_dropping_indices(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(name);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Ball(size);") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Ball;") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}) YIELD * RETURN *;")) + assert len(results) == 4 + assert results[0] == ("Dropped", "", [], "Ball", False) + assert results[1] == ("Dropped", "size", ["size"], "Ball", False) + assert results[2] == ("Dropped", "id", ["id"], "Person", False) + assert results[3] == ("Dropped", "name", ["name"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [] + + +def test_assert_existence_constraint_properties_not_list(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: 'name'}) YIELD * RETURN *;") + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + + +def test_assert_existence_constraint_property_not_string(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: ['name', 1]}) YIELD * RETURN *;") + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name")] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + + +def test_assert_existence_constraint_property_empty_string(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {Person: ['']}) YIELD * RETURN *;") + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + + +def test_assert_creates_indices_and_existence_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id']}, {}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert len(results) == 4 + assert results[0] == ("Created", "", [], "Person", False) + assert results[1] == ("Created", "id", ["id"], "Person", False) + assert results[2] == ("Created", "name", ["name"], "Person", False) + assert results[3] == ("Created", "surname", ["surname"], "Person", False) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("exists", "Person", "name"), ("exists", "Person", "surname")] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_drops_existence_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 2 + assert results[0] == ("Dropped", "name", ["name"], "Person", False) + assert results[1] == ("Dropped", "surname", ["surname"], "Person", False) + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_creates_unique_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname']]}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_multiple_unique_constraints(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname'], ['id']]}) YIELD * RETURN *;", + ) + ) + assert results == [ + ("Created", "[name, surname]", ["name", "surname"], "Person", True), + ("Created", "[id]", ["id"], "Person", True), + ] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"]), ("unique", "Person", ["id"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + + +def test_assert_creates_unique_constraints_skip_invalid(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname'], 'wrong_type']}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_unique_constraints_skip_invalid_map_type(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({}, {Person: [['name', 'surname']], Ball: 'wrong_type'}) YIELD * RETURN *;", + ) + ) + assert results == [("Created", "[name, surname]", ["name", "surname"], "Person", True)] + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [("unique", "Person", ["name", "surname"])] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + + +def test_assert_creates_constraints_and_indices(): + cursor = connect().cursor() + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['', 'id']}, {Person: [['name', 'surname'], ['id']]}, {Person: ['name', 'surname']}) YIELD * RETURN *;", + ) + ) + assert len(results) == 6 + assert results[0] == ("Created", "", [], "Person", False) + assert results[1] == ("Created", "id", ["id"], "Person", False) + assert results[2] == ("Created", "name", ["name"], "Person", False) + assert results[3] == ("Created", "surname", ["surname"], "Person", False) + assert results[4] == ("Created", "[name, surname]", ["name", "surname"], "Person", True) + assert results[5] == ("Created", "[id]", ["id"], "Person", True) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [ + ("exists", "Person", "name"), + ("exists", "Person", "surname"), + ("unique", "Person", ["name", "surname"]), + ("unique", "Person", ["id"]), + ] + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_drops_unique_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 2 + assert results[0] == ("Dropped", "[id]", ["id"], "Person", True) + assert results[1] == ("Dropped", "[name, surname]", ["name", "surname"], "Person", True) + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_drops_indices_and_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}) YIELD * RETURN *;")) + assert len(results) == 6 + assert results[0] == ("Dropped", "", [], "Person", False) + assert results[1] == ("Dropped", "id", ["id"], "Person", False) + assert results[2] == ("Dropped", "name", ["name"], "Person", False) + assert results[3] == ("Dropped", "surname", ["surname"], "Person", False) + assert results[4] == ("Dropped", "[id]", ["id"], "Person", True) + assert results[5] == ("Dropped", "[name, surname]", ["name", "surname"], "Person", True) + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [] + + +def test_assert_does_not_drop_indices_and_constraints(): + cursor = connect().cursor() + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list(execute_and_fetch_all(cursor, "CALL libschema.assert({}, {}, {}, false) YIELD * RETURN *;")) + assert len(results) == 0 + show_index_results = list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) + assert show_index_results == [("label", "Person", None, 0), ("label+property", "Person", "id", 0)] + show_constraint_results = list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) + assert show_constraint_results == [ + ("exists", "Person", "name"), + ("exists", "Person", "surname"), + ("unique", "Person", ["name", "surname"]), + ("unique", "Person", ["id"]), + ] + execute_and_fetch_all(cursor, "DROP INDEX ON :Person;") + execute_and_fetch_all(cursor, "DROP INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "DROP CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + + +def test_assert_keeps_existing_indices_and_constraints(): + cursor = connect().cursor() + assert list(execute_and_fetch_all(cursor, "SHOW INDEX INFO;")) == [] + assert list(execute_and_fetch_all(cursor, "SHOW CONSTRAINT INFO;")) == [] + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person;") + execute_and_fetch_all(cursor, "CREATE INDEX ON :Person(id);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.name, n.surname IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT n.id IS UNIQUE;") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.name);") + execute_and_fetch_all(cursor, "CREATE CONSTRAINT ON (n:Person) ASSERT EXISTS (n.surname);") + results = list( + execute_and_fetch_all( + cursor, + "CALL libschema.assert({Person: ['id']}, {Person: [['name', 'surname']]}, {Person: ['name']}) YIELD * RETURN *;", + ) + ) + + print(results) + + assert len(results) == 6 + + assert results[0] == ("Kept", "id", ["id"], "Person", False) # label+property index on Person(id) should be kept + assert results[1] == ("Dropped", "", [], "Person", False) # label index on Person should be deleted + assert results[2] == ( + "Kept", + "name", + ["name"], + "Person", + False, + ) # existence constraint on Person(name) should be kept + assert results[3] == ( + "Dropped", + "surname", + ["surname"], + "Person", + False, + ) # existence constraint on surname should be deleted + assert results[4] == ( + "Kept", + "[name, surname]", + ["name", "surname"], + "Person", + True, + ) # unique constraint on Person(name, surname) should be kept + assert results[5] == ( + "Dropped", + "[id]", + ["id"], + "Person", + True, + ) # unique constraint on Person(id) should be deleted + + def test_node_type_properties1(): cursor = connect().cursor() execute_and_fetch_all( diff --git a/tests/e2e/query_modules/workloads.yaml b/tests/e2e/query_modules/workloads.yaml index ae6c45c90..54641c3d4 100644 --- a/tests/e2e/query_modules/workloads.yaml +++ b/tests/e2e/query_modules/workloads.yaml @@ -27,7 +27,7 @@ workloads: args: ["query_modules/mgps_test.py"] <<: *in_memory_cluster - - name: "Convert query module test" + - name: "Schema test" pre_set_workload: "tests/e2e/x.sh" binary: "tests/e2e/pytest_runner.sh" proc: "query_modules/" From e4f94c15c6ef9dcee26d98c977744331b51db4a1 Mon Sep 17 00:00:00 2001 From: Gareth Andrew Lloyd Date: Wed, 22 Nov 2023 13:05:02 +0000 Subject: [PATCH 08/10] Fixes for clang-tidy / sonar issues (#1536) --- include/mgp.hpp | 292 +++++++++--------- query_modules/schema.cpp | 2 - src/audit/log.cpp | 5 +- src/audit/log.hpp | 2 +- src/auth/models.cpp | 13 +- src/auth/models.hpp | 26 +- src/communication/bolt/client.hpp | 5 +- .../v1/decoder/chunked_decoder_buffer.hpp | 4 +- src/communication/bolt/v1/decoder/decoder.hpp | 6 +- .../bolt/v1/encoder/base_encoder.hpp | 10 +- .../v1/encoder/chunked_encoder_buffer.hpp | 4 +- .../bolt/v1/encoder/client_encoder.hpp | 2 +- src/communication/bolt/v1/encoder/encoder.hpp | 2 +- src/communication/bolt/v1/states/handlers.hpp | 6 +- src/communication/bolt/v1/states/init.hpp | 8 +- src/communication/bolt/v1/value.hpp | 2 +- src/communication/client.hpp | 4 +- src/communication/context.cpp | 6 +- src/communication/helpers.cpp | 4 +- src/communication/helpers.hpp | 4 +- src/communication/http/listener.hpp | 2 +- src/communication/listener.hpp | 5 +- src/communication/result_stream_faker.hpp | 4 +- src/communication/session.hpp | 4 +- src/communication/v2/listener.hpp | 2 +- src/communication/v2/session.hpp | 2 +- src/communication/websocket/server.hpp | 5 +- src/dbms/dbms_handler.hpp | 2 +- src/dbms/handler.hpp | 2 +- src/glue/SessionHL.cpp | 5 +- src/glue/SessionHL.hpp | 2 +- src/glue/communication.cpp | 4 +- src/helpers.hpp | 2 +- src/http_handlers/metrics.hpp | 1 - src/integrations/kafka/consumer.hpp | 4 +- src/integrations/pulsar/consumer.cpp | 4 +- src/io/network/endpoint.hpp | 10 +- src/io/network/epoll.hpp | 4 +- src/io/network/socket.hpp | 3 +- src/kvstore/kvstore.cpp | 2 +- src/query/common.hpp | 6 +- src/query/cypher_query_interpreter.hpp | 6 +- src/query/db_accessor.hpp | 2 - src/query/dump.cpp | 4 +- src/query/frame_change.hpp | 4 +- src/query/frontend/ast/ast.hpp | 15 +- src/query/frontend/ast/pretty_print.cpp | 2 +- src/query/frontend/opencypher/parser.hpp | 4 +- src/query/frontend/semantic/symbol.hpp | 19 +- .../frontend/semantic/symbol_generator.hpp | 2 +- src/query/frontend/semantic/symbol_table.hpp | 2 +- src/query/frontend/stripped.cpp | 37 +-- src/query/frontend/stripped.hpp | 4 +- .../frontend/stripped_lexer_constants.hpp | 8 +- src/query/interpret/eval.hpp | 4 +- src/query/interpreter.cpp | 31 +- src/query/interpreter.hpp | 2 +- src/query/plan/operator.cpp | 81 ++--- src/query/plan/operator.hpp | 111 ++++--- src/query/plan/planner.hpp | 8 +- src/query/plan/preprocess.cpp | 13 +- src/query/plan/preprocess.hpp | 48 +-- src/query/plan/profile.hpp | 7 +- src/query/plan/rewrite/index_lookup.hpp | 7 +- src/query/plan/variable_start_planner.cpp | 18 +- src/query/plan/variable_start_planner.hpp | 38 +-- src/query/plan/vertex_count_cache.hpp | 9 +- .../include/replication/config.hpp | 4 +- .../replication/replication_server.hpp | 4 +- src/requests/requests.cpp | 2 +- src/rpc/client.hpp | 2 +- src/rpc/client_pool.hpp | 7 +- src/rpc/exceptions.hpp | 2 +- src/rpc/protocol.cpp | 6 +- src/rpc/protocol.hpp | 2 +- src/slk/serialization.hpp | 20 +- src/slk/streams.cpp | 5 +- src/storage/v2/disk/label_property_index.cpp | 3 +- src/storage/v2/disk/rocksdb_storage.hpp | 2 +- src/storage/v2/disk/unique_constraints.cpp | 2 +- src/storage/v2/durability/serialization.hpp | 6 +- src/storage/v2/durability/snapshot.cpp | 4 +- src/storage/v2/durability/wal.hpp | 2 +- src/storage/v2/edge_accessor.hpp | 2 +- src/storage/v2/id_types.hpp | 2 +- src/storage/v2/modified_edge.hpp | 3 +- src/storage/v2/property_store.cpp | 8 +- .../v2/replication/replication_client.hpp | 2 +- src/storage/v2/replication/rpc.cpp | 7 +- src/storage/v2/replication/rpc.hpp | 38 +-- src/storage/v2/storage.hpp | 2 +- src/storage/v2/vertex_accessor.hpp | 2 +- src/telemetry/telemetry.cpp | 5 +- src/utils/cast.hpp | 10 +- src/utils/event_histogram.hpp | 7 +- src/utils/exceptions.hpp | 4 +- src/utils/logging.hpp | 22 +- src/utils/lru_cache.hpp | 2 +- src/utils/memory.hpp | 4 +- src/utils/scheduler.hpp | 2 +- src/utils/skip_list.hpp | 4 +- src/utils/small_vector.hpp | 42 +-- src/utils/stacktrace.hpp | 11 +- src/utils/variant_helpers.hpp | 2 +- tests/benchmark/query/profile.cpp | 14 +- tests/benchmark/rpc.cpp | 5 +- .../e2e/isolation_levels/isolation_levels.cpp | 2 +- .../memory/memory_limit_global_alloc_proc.cpp | 2 +- ..._limit_global_multi_thread_proc_create.cpp | 2 +- .../memory_limit_global_thread_alloc_proc.cpp | 2 +- tests/e2e/memory/procedure_memory_limit.cpp | 2 +- tests/e2e/memory/query_memory_limit_proc.cpp | 2 +- .../query_memory_limit_proc_multi_thread.cpp | 2 +- tests/e2e/replication/common.hpp | 4 +- tests/e2e/replication/constraints.cpp | 4 +- .../e2e/replication/read_write_benchmark.cpp | 4 +- tests/e2e/triggers/common.hpp | 4 +- tests/manual/antlr_tree_pretty_print.cpp | 4 +- tests/stress/long_running.cpp | 4 +- tests/unit/CMakeLists.txt | 17 +- tests/unit/auth.cpp | 20 +- tests/unit/auth_handler.cpp | 4 +- tests/unit/bfs_common.hpp | 2 +- tests/unit/bolt_chunked_encoder_buffer.cpp | 6 +- tests/unit/cpp_api.cpp | 2 +- tests/unit/cypher_main_visitor.cpp | 6 +- tests/unit/integrations_kafka_consumer.cpp | 6 +- tests/unit/interpreter.cpp | 4 +- tests/unit/kvstore.cpp | 6 +- tests/unit/mgp_kafka_c_api.cpp | 4 +- tests/unit/plan_pretty_print.cpp | 2 +- tests/unit/query_common.hpp | 22 +- tests/unit/query_cost_estimator.cpp | 12 +- tests/unit/query_dump.cpp | 8 +- tests/unit/query_expression_evaluator.cpp | 8 +- tests/unit/query_plan_checker.hpp | 17 +- tests/unit/query_plan_common.hpp | 14 +- tests/unit/query_plan_edge_cases.cpp | 4 +- tests/unit/query_plan_operator_to_string.cpp | 2 +- tests/unit/query_required_privileges.cpp | 2 +- tests/unit/query_semantic.cpp | 54 ++-- tests/unit/rpc_messages.hpp | 12 +- tests/unit/slk_core.cpp | 6 +- tests/unit/slk_streams.cpp | 4 +- tests/unit/storage_rocks.cpp | 2 +- tests/unit/storage_v2_property_store.cpp | 2 +- tests/unit/storage_v2_wal_file.cpp | 6 +- tests/unit/utils_file.cpp | 4 +- tests/unit/utils_string.cpp | 4 +- 149 files changed, 778 insertions(+), 763 deletions(-) diff --git a/include/mgp.hpp b/include/mgp.hpp index 25e365e53..6296d2e5c 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -34,7 +34,7 @@ namespace mgp { class IndexException : public std::exception { public: - explicit IndexException(const std::string &message) : message_(message) {} + explicit IndexException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -43,7 +43,7 @@ class IndexException : public std::exception { class ValueException : public std::exception { public: - explicit ValueException(const std::string &message) : message_(message) {} + explicit ValueException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -52,7 +52,7 @@ class ValueException : public std::exception { class NotFoundException : public std::exception { public: - explicit NotFoundException(const std::string &message) : message_(message) {} + explicit NotFoundException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -61,7 +61,7 @@ class NotFoundException : public std::exception { class MustAbortException : public std::exception { public: - explicit MustAbortException(const std::string &message) : message_(message) {} + explicit MustAbortException(std::string message) : message_(std::move(message)) {} const char *what() const noexcept override { return message_.c_str(); } private: @@ -233,14 +233,14 @@ class Graph { GraphRelationships Relationships() const; /// @brief Returns the graph node with the given ID. - Node GetNodeById(const Id node_id) const; + Node GetNodeById(Id node_id) const; /// @brief Returns whether the graph contains a node with the given ID. - bool ContainsNode(const Id node_id) const; + bool ContainsNode(Id node_id) const; /// @brief Returns whether the graph contains the given node. bool ContainsNode(const Node &node) const; /// @brief Returns whether the graph contains a relationship with the given ID. - bool ContainsRelationship(const Id relationship_id) const; + bool ContainsRelationship(Id relationship_id) const; /// @brief Returns whether the graph contains the given relationship. bool ContainsRelationship(const Relationship &relationship) const; @@ -253,7 +253,7 @@ class Graph { /// @brief Deletes a node and all its incident edges from the graph. void DetachDeleteNode(const Node &node); /// @brief Creates a relationship of type `type` between nodes `from` and `to` and adds it to the graph. - Relationship CreateRelationship(const Node &from, const Node &to, const std::string_view type); + Relationship CreateRelationship(const Node &from, const Node &to, std::string_view type); /// @brief Changes a relationship from node. void SetFrom(Relationship &relationship, const Node &new_from); /// @brief Changes a relationship to node. @@ -305,7 +305,7 @@ class Nodes { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Node operator*() const; + Node operator*() const; private: mgp_vertices_iterator *nodes_iterator_ = nullptr; @@ -352,7 +352,7 @@ class GraphRelationships { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Relationship operator*() const; + Relationship operator*() const; private: mgp_vertices_iterator *nodes_iterator_ = nullptr; @@ -398,7 +398,7 @@ class Relationships { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const Relationship operator*() const; + Relationship operator*() const; private: mgp_edges_iterator *relationships_iterator_ = nullptr; @@ -451,7 +451,7 @@ class Labels { Iterator &operator++(); - const std::string_view operator*() const; + std::string_view operator*() const; private: Iterator(const Labels *iterable, size_t index); @@ -502,7 +502,7 @@ class List { explicit List(std::vector &&values); /// @brief Creates a List from the given initializer_list. - explicit List(const std::initializer_list list); + explicit List(std::initializer_list list); List(const List &other) noexcept; List(List &&other) noexcept; @@ -518,7 +518,7 @@ class List { bool Empty() const; /// @brief Returns the value at the given `index`. - const Value operator[](size_t index) const; + Value operator[](size_t index) const; ///@brief Same as above, but non const value Value operator[](size_t index); @@ -540,7 +540,7 @@ class List { Iterator &operator++(); - const Value operator*() const; + Value operator*() const; private: Iterator(const List *iterable, size_t index); @@ -577,7 +577,7 @@ class List { bool operator!=(const List &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_list *ptr_; @@ -608,7 +608,7 @@ class Map { explicit Map(std::map &&items); /// @brief Creates a Map from the given initializer_list (map items correspond to initializer list pairs). - Map(const std::initializer_list> items); + Map(std::initializer_list> items); Map(const Map &other) noexcept; Map(Map &&other) noexcept; @@ -625,10 +625,10 @@ class Map { bool Empty() const; /// @brief Returns the value at the given `key`. - Value const operator[](std::string_view key) const; + Value operator[](std::string_view key) const; /// @brief Returns the value at the given `key`. - Value const At(std::string_view key) const; + Value At(std::string_view key) const; /// @brief Returns true if the given `key` exists. bool KeyExists(std::string_view key) const; @@ -656,7 +656,7 @@ class Map { bool operator==(Iterator other) const; bool operator!=(Iterator other) const; - const MapItem operator*() const; + MapItem operator*() const; private: mgp_map_items_iterator *map_items_iterator_ = nullptr; @@ -698,7 +698,7 @@ class Map { bool operator!=(const Map &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_map *ptr_; @@ -761,10 +761,10 @@ class Node { Relationships OutRelationships() const; /// @brief Adds a label to the node. - void AddLabel(const std::string_view label); + void AddLabel(std::string_view label); /// @brief Removes a label from the node. - void RemoveLabel(const std::string_view label); + void RemoveLabel(std::string_view label); bool operator<(const Node &other) const; @@ -775,7 +775,7 @@ class Node { bool operator!=(const Node &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; /// @brief returns the in degree of a node inline size_t InDegree() const; @@ -845,7 +845,7 @@ class Relationship { bool operator!=(const Relationship &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_edge *ptr_; @@ -898,7 +898,7 @@ class Path { bool operator!=(const Path &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_path *ptr_; @@ -958,7 +958,7 @@ class Date { bool operator<(const Date &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_date *ptr_; @@ -1020,7 +1020,7 @@ class LocalTime { bool operator<(const LocalTime &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_local_time *ptr_; @@ -1088,7 +1088,7 @@ class LocalDateTime { bool operator<(const LocalDateTime &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_local_date_time *ptr_; @@ -1142,7 +1142,7 @@ class Duration { bool operator<(const Duration &other) const; /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_duration *ptr_; @@ -1190,13 +1190,13 @@ class Value { explicit Value(); // Primitive type constructors: - explicit Value(const bool value); - explicit Value(const int64_t value); - explicit Value(const double value); + explicit Value(bool value); + explicit Value(int64_t value); + explicit Value(double value); // String constructors: explicit Value(const char *value); - explicit Value(const std::string_view value); + explicit Value(std::string_view value); // Container constructors: /// @brief Constructs a List value from the copy of the given `list`. @@ -1290,28 +1290,28 @@ class Value { List ValueList() const; List ValueList(); /// @pre Value type needs to be Type::Map. - const Map ValueMap() const; + Map ValueMap() const; Map ValueMap(); /// @pre Value type needs to be Type::Node. - const Node ValueNode() const; + Node ValueNode() const; Node ValueNode(); /// @pre Value type needs to be Type::Relationship. - const Relationship ValueRelationship() const; + Relationship ValueRelationship() const; Relationship ValueRelationship(); /// @pre Value type needs to be Type::Path. - const Path ValuePath() const; + Path ValuePath() const; Path ValuePath(); /// @pre Value type needs to be Type::Date. - const Date ValueDate() const; + Date ValueDate() const; Date ValueDate(); /// @pre Value type needs to be Type::LocalTime. - const LocalTime ValueLocalTime() const; + LocalTime ValueLocalTime() const; LocalTime ValueLocalTime(); /// @pre Value type needs to be Type::LocalDateTime. - const LocalDateTime ValueLocalDateTime() const; + LocalDateTime ValueLocalDateTime() const; LocalDateTime ValueLocalDateTime(); /// @pre Value type needs to be Type::Duration. - const Duration ValueDuration() const; + Duration ValueDuration() const; Duration ValueDuration(); /// @brief Returns whether the value is null. @@ -1355,7 +1355,7 @@ class Value { friend std::ostream &operator<<(std::ostream &os, const mgp::Value &value); /// @brief returns the string representation - const std::string ToString() const; + std::string ToString() const; private: mgp_value *ptr_; @@ -1421,9 +1421,9 @@ class RecordFactory { public: explicit RecordFactory(mgp_result *result); - const Record NewRecord() const; + Record NewRecord() const; - void SetErrorMessage(const std::string_view error_msg) const; + void SetErrorMessage(std::string_view error_msg) const; void SetErrorMessage(const char *error_msg) const; @@ -1465,7 +1465,7 @@ class Result { /// @brief Sets a @ref Duration value to be returned. inline void SetValue(const Duration &duration); - void SetErrorMessage(const std::string_view error_msg) const; + void SetErrorMessage(std::string_view error_msg) const; void SetErrorMessage(const char *error_msg) const; @@ -1644,8 +1644,8 @@ template TDest MemcpyCast(TSrc src) { TDest dest; static_assert(sizeof(dest) == sizeof(src), "MemcpyCast expects source and destination to be of the same size"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects source to be an arithmetic type"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects destination to be an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects source to be an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects destination to be an arithmetic type"); std::memcpy(&dest, &src, sizeof(src)); return dest; } @@ -1678,8 +1678,8 @@ inline bool MapsEqual(mgp_map *map1, mgp_map *map2) { if (mgp::map_size(map1) != mgp::map_size(map2)) { return false; } - auto items_it = mgp::MemHandlerCallback(map_iter_items, map1); - for (auto item = mgp::map_items_iterator_get(items_it); item; item = mgp::map_items_iterator_next(items_it)) { + auto *items_it = mgp::MemHandlerCallback(map_iter_items, map1); + for (auto *item = mgp::map_items_iterator_get(items_it); item; item = mgp::map_items_iterator_next(items_it)) { if (mgp::map_item_key(item) == mgp::map_item_key(item)) { return false; } @@ -1943,7 +1943,7 @@ inline int64_t Graph::Size() const { } inline GraphNodes Graph::Nodes() const { - auto nodes_it = mgp::MemHandlerCallback(graph_iter_vertices, graph_); + auto *nodes_it = mgp::MemHandlerCallback(graph_iter_vertices, graph_); if (nodes_it == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -1953,7 +1953,7 @@ inline GraphNodes Graph::Nodes() const { inline GraphRelationships Graph::Relationships() const { return GraphRelationships(graph_); } inline Node Graph::GetNodeById(const Id node_id) const { - auto mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); + auto *mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); if (mgp_node == nullptr) { mgp::vertex_destroy(mgp_node); throw NotFoundException("Node with ID " + std::to_string(node_id.AsUint()) + " not found!"); @@ -1964,7 +1964,7 @@ inline Node Graph::GetNodeById(const Id node_id) const { } inline bool Graph::ContainsNode(const Id node_id) const { - auto mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); + auto *mgp_node = mgp::MemHandlerCallback(graph_get_vertex_by_id, graph_, mgp_vertex_id{.as_int = node_id.AsInt()}); if (mgp_node == nullptr) { return false; } @@ -2066,7 +2066,7 @@ inline Nodes::Iterator::~Iterator() { inline Nodes::Iterator &Nodes::Iterator::operator++() { if (nodes_iterator_ != nullptr) { - auto next = mgp::vertices_iterator_next(nodes_iterator_); + auto *next = mgp::vertices_iterator_next(nodes_iterator_); if (next == nullptr) { mgp::vertices_iterator_destroy(nodes_iterator_); @@ -2098,7 +2098,7 @@ inline bool Nodes::Iterator::operator==(Iterator other) const { inline bool Nodes::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Node Nodes::Iterator::operator*() const { +inline Node Nodes::Iterator::operator*() const { if (nodes_iterator_ == nullptr) { return Node((const mgp_vertex *)nullptr); } @@ -2126,7 +2126,7 @@ inline GraphRelationships::Iterator::Iterator(mgp_vertices_iterator *nodes_itera } // Go through each graph node’s adjacent nodes - for (auto node = mgp::vertices_iterator_get(nodes_iterator_); node; + for (auto *node = mgp::vertices_iterator_get(nodes_iterator_); node; node = mgp::vertices_iterator_next(nodes_iterator_)) { // Check if node exists if (node == nullptr) { @@ -2137,7 +2137,7 @@ inline GraphRelationships::Iterator::Iterator(mgp_vertices_iterator *nodes_itera // Check if node has out-relationships out_relationships_iterator_ = mgp::MemHandlerCallback(vertex_iter_out_edges, node); - auto relationship = mgp::edges_iterator_get(out_relationships_iterator_); + auto *relationship = mgp::edges_iterator_get(out_relationships_iterator_); if (relationship != nullptr) { return; } @@ -2164,7 +2164,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // 1. Check if the current node has remaining relationships to iterate over if (out_relationships_iterator_ != nullptr) { - auto next = mgp::edges_iterator_next(out_relationships_iterator_); + auto *next = mgp::edges_iterator_next(out_relationships_iterator_); if (next != nullptr) { return *this; @@ -2177,7 +2177,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // 2. Move onto the next nodes if (nodes_iterator_ != nullptr) { - for (auto node = mgp::vertices_iterator_next(nodes_iterator_); node; + for (auto *node = mgp::vertices_iterator_next(nodes_iterator_); node; node = mgp::vertices_iterator_next(nodes_iterator_)) { // Check if node exists - if it doesn’t, we’ve reached the end of the iterator if (node == nullptr) { @@ -2188,7 +2188,7 @@ inline GraphRelationships::Iterator &GraphRelationships::Iterator::operator++() // Check if node has out-relationships out_relationships_iterator_ = mgp::MemHandlerCallback(vertex_iter_out_edges, node); - auto relationship = mgp::edges_iterator_get(out_relationships_iterator_); + auto *relationship = mgp::edges_iterator_get(out_relationships_iterator_); if (relationship != nullptr) { return *this; } @@ -2222,7 +2222,7 @@ inline bool GraphRelationships::Iterator::operator==(Iterator other) const { inline bool GraphRelationships::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Relationship GraphRelationships::Iterator::operator*() const { +inline Relationship GraphRelationships::Iterator::operator*() const { if (out_relationships_iterator_ != nullptr) { return Relationship(mgp::edges_iterator_get(out_relationships_iterator_)); } @@ -2268,7 +2268,7 @@ inline Relationships::Iterator::~Iterator() { inline Relationships::Iterator &Relationships::Iterator::operator++() { if (relationships_iterator_ != nullptr) { - auto next = mgp::edges_iterator_next(relationships_iterator_); + auto *next = mgp::edges_iterator_next(relationships_iterator_); if (next == nullptr) { mgp::edges_iterator_destroy(relationships_iterator_); @@ -2300,7 +2300,7 @@ inline bool Relationships::Iterator::operator==(Iterator other) const { inline bool Relationships::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const Relationship Relationships::Iterator::operator*() const { +inline Relationship Relationships::Iterator::operator*() const { if (relationships_iterator_ == nullptr) { return Relationship((mgp_edge *)nullptr); } @@ -2361,7 +2361,7 @@ inline Labels::Iterator &Labels::Iterator::operator++() { return *this; } -inline const std::string_view Labels::Iterator::operator*() const { return (*iterable_)[index_]; } +inline std::string_view Labels::Iterator::operator*() const { return (*iterable_)[index_]; } inline Labels::Iterator::Iterator(const Labels *iterable, size_t index) : iterable_(iterable), index_(index) {} @@ -2446,7 +2446,7 @@ inline size_t List::Size() const { return mgp::list_size(ptr_); } inline bool List::Empty() const { return Size() == 0; } -inline const Value List::operator[](size_t index) const { return Value(mgp::list_at(ptr_, index)); } +inline Value List::operator[](size_t index) const { return Value(mgp::list_at(ptr_, index)); } inline Value List::operator[](size_t index) { return Value(mgp::list_at(ptr_, index)); } @@ -2461,7 +2461,7 @@ inline List::Iterator &List::Iterator::operator++() { return *this; } -inline const Value List::Iterator::operator*() const { return (*iterable_)[index_]; } +inline Value List::Iterator::operator*() const { return (*iterable_)[index_]; } inline List::Iterator::Iterator(const List *iterable, size_t index) : iterable_(iterable), index_(index) {} @@ -2488,7 +2488,7 @@ inline bool List::operator==(const List &other) const { return util::ListsEqual( inline bool List::operator!=(const List &other) const { return !(*this == other); } -inline const std::string List::ToString() const { +inline std::string List::ToString() const { const size_t size = Size(); if (size == 0) { return "[]"; @@ -2572,9 +2572,9 @@ inline size_t Map::Size() const { return mgp::map_size(ptr_); } inline bool Map::Empty() const { return Size() == 0; } -inline const Value Map::operator[](std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); } +inline Value Map::operator[](std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); } -inline const Value Map::At(std::string_view key) const { +inline Value Map::At(std::string_view key) const { auto *ptr = mgp::map_at(ptr_, key.data()); if (ptr) { return Value(ptr); @@ -2603,7 +2603,7 @@ inline Map::Iterator::~Iterator() { inline Map::Iterator &Map::Iterator::operator++() { if (map_items_iterator_ != nullptr) { - auto next = mgp::map_items_iterator_next(map_items_iterator_); + auto *next = mgp::map_items_iterator_next(map_items_iterator_); if (next == nullptr) { mgp::map_items_iterator_destroy(map_items_iterator_); @@ -2632,14 +2632,14 @@ inline bool Map::Iterator::operator==(Iterator other) const { inline bool Map::Iterator::operator!=(Iterator other) const { return !(*this == other); } -inline const MapItem Map::Iterator::operator*() const { +inline MapItem Map::Iterator::operator*() const { if (map_items_iterator_ == nullptr) { throw ValueException("Empty map item!"); } - auto raw_map_item = mgp::map_items_iterator_get(map_items_iterator_); + auto *raw_map_item = mgp::map_items_iterator_get(map_items_iterator_); - auto map_key = mgp::map_item_key(raw_map_item); + const auto *map_key = mgp::map_item_key(raw_map_item); auto map_value = Value(mgp::map_item_value(raw_map_item)); return MapItem{.key = map_key, .value = map_value}; @@ -2675,7 +2675,7 @@ inline bool Map::operator==(const Map &other) const { return util::MapsEqual(ptr inline bool Map::operator!=(const Map &other) const { return !(*this == other); } -inline const std::string Map::ToString() const { +inline std::string Map::ToString() const { const size_t map_size = Size(); if (map_size == 0) { return "{}"; @@ -2747,7 +2747,7 @@ inline bool Node::HasLabel(std::string_view label) const { } inline Relationships Node::InRelationships() const { - auto relationship_iterator = mgp::MemHandlerCallback(vertex_iter_in_edges, ptr_); + auto *relationship_iterator = mgp::MemHandlerCallback(vertex_iter_in_edges, ptr_); if (relationship_iterator == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -2755,7 +2755,7 @@ inline Relationships Node::InRelationships() const { } inline Relationships Node::OutRelationships() const { - auto relationship_iterator = mgp::MemHandlerCallback(vertex_iter_out_edges, ptr_); + auto *relationship_iterator = mgp::MemHandlerCallback(vertex_iter_out_edges, ptr_); if (relationship_iterator == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -2825,7 +2825,7 @@ inline std::string PropertiesToString(const std::map &proper return properties; } -inline const std::string Node::ToString() const { +inline std::string Node::ToString() const { std::string labels{", "}; for (auto label : Labels()) { labels.append(":" + std::string(label)); @@ -2933,7 +2933,7 @@ inline bool Relationship::operator==(const Relationship &other) const { inline bool Relationship::operator!=(const Relationship &other) const { return !(*this == other); } -inline const std::string Relationship::ToString() const { +inline std::string Relationship::ToString() const { const auto from = From(); const auto to = To(); @@ -2992,7 +2992,7 @@ inline Path::~Path() { inline size_t Path::Length() const { return mgp::path_size(ptr_); } inline Node Path::GetNodeAt(size_t index) const { - auto node_ptr = mgp::path_vertex_at(ptr_, index); + auto *node_ptr = mgp::path_vertex_at(ptr_, index); if (node_ptr == nullptr) { throw IndexException("Index value out of bounds."); } @@ -3000,7 +3000,7 @@ inline Node Path::GetNodeAt(size_t index) const { } inline Relationship Path::GetRelationshipAt(size_t index) const { - auto relationship_ptr = mgp::path_edge_at(ptr_, index); + auto *relationship_ptr = mgp::path_edge_at(ptr_, index); if (relationship_ptr == nullptr) { throw IndexException("Index value out of bounds."); } @@ -3015,10 +3015,10 @@ inline bool Path::operator==(const Path &other) const { return util::PathsEqual( inline bool Path::operator!=(const Path &other) const { return !(*this == other); } -inline const std::string Path::ToString() const { +inline std::string Path::ToString() const { const auto length = Length(); size_t i = 0; - std::string return_string{""}; + std::string return_string; for (i = 0; i < length; i++) { const auto node = GetNodeAt(i); return_string.append(node.ToString() + "-"); @@ -3089,7 +3089,7 @@ inline Date::~Date() { } inline Date Date::Now() { - auto mgp_date = mgp::MemHandlerCallback(date_now); + auto *mgp_date = mgp::MemHandlerCallback(date_now); auto date = Date(mgp_date); mgp::date_destroy(mgp_date); @@ -3107,7 +3107,7 @@ inline int64_t Date::Timestamp() const { return mgp::date_timestamp(ptr_); } inline bool Date::operator==(const Date &other) const { return util::DatesEqual(ptr_, other.ptr_); } inline Date Date::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); auto sum = Date(mgp_sum); mgp::date_destroy(mgp_sum); @@ -3115,7 +3115,7 @@ inline Date Date::operator+(const Duration &dur) const { } inline Date Date::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(date_add_duration, ptr_, dur.ptr_); auto difference = Date(mgp_difference); mgp::date_destroy(mgp_difference); @@ -3123,7 +3123,7 @@ inline Date Date::operator-(const Duration &dur) const { } inline Duration Date::operator-(const Date &other) const { - auto mgp_difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3131,14 +3131,14 @@ inline Duration Date::operator-(const Date &other) const { } inline bool Date::operator<(const Date &other) const { - auto difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(date_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string Date::ToString() const { +inline std::string Date::ToString() const { return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()); } @@ -3188,7 +3188,7 @@ inline LocalTime::~LocalTime() { } inline LocalTime LocalTime::Now() { - auto mgp_local_time = mgp::MemHandlerCallback(local_time_now); + auto *mgp_local_time = mgp::MemHandlerCallback(local_time_now); auto local_time = LocalTime(mgp_local_time); mgp::local_time_destroy(mgp_local_time); @@ -3210,7 +3210,7 @@ inline int64_t LocalTime::Timestamp() const { return mgp::local_time_timestamp(p inline bool LocalTime::operator==(const LocalTime &other) const { return util::LocalTimesEqual(ptr_, other.ptr_); } inline LocalTime LocalTime::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(local_time_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(local_time_add_duration, ptr_, dur.ptr_); auto sum = LocalTime(mgp_sum); mgp::local_time_destroy(mgp_sum); @@ -3218,7 +3218,7 @@ inline LocalTime LocalTime::operator+(const Duration &dur) const { } inline LocalTime LocalTime::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(local_time_sub_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_time_sub_duration, ptr_, dur.ptr_); auto difference = LocalTime(mgp_difference); mgp::local_time_destroy(mgp_difference); @@ -3226,7 +3226,7 @@ inline LocalTime LocalTime::operator-(const Duration &dur) const { } inline Duration LocalTime::operator-(const LocalTime &other) const { - auto mgp_difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3234,14 +3234,14 @@ inline Duration LocalTime::operator-(const LocalTime &other) const { } inline bool LocalTime::operator<(const LocalTime &other) const { - auto difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(local_time_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string LocalTime::ToString() const { +inline std::string LocalTime::ToString() const { return std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + std::to_string(Millisecond()) + std::to_string(Microsecond()); } @@ -3299,7 +3299,7 @@ inline LocalDateTime::~LocalDateTime() { } inline LocalDateTime LocalDateTime::Now() { - auto mgp_local_date_time = mgp::MemHandlerCallback(local_date_time_now); + auto *mgp_local_date_time = mgp::MemHandlerCallback(local_date_time_now); auto local_date_time = LocalDateTime(mgp_local_date_time); mgp::local_date_time_destroy(mgp_local_date_time); @@ -3329,7 +3329,7 @@ inline bool LocalDateTime::operator==(const LocalDateTime &other) const { } inline LocalDateTime LocalDateTime::operator+(const Duration &dur) const { - auto mgp_sum = mgp::MemHandlerCallback(local_date_time_add_duration, ptr_, dur.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(local_date_time_add_duration, ptr_, dur.ptr_); auto sum = LocalDateTime(mgp_sum); mgp::local_date_time_destroy(mgp_sum); @@ -3337,7 +3337,7 @@ inline LocalDateTime LocalDateTime::operator+(const Duration &dur) const { } inline LocalDateTime LocalDateTime::operator-(const Duration &dur) const { - auto mgp_difference = mgp::MemHandlerCallback(local_date_time_sub_duration, ptr_, dur.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_date_time_sub_duration, ptr_, dur.ptr_); auto difference = LocalDateTime(mgp_difference); mgp::local_date_time_destroy(mgp_difference); @@ -3345,7 +3345,7 @@ inline LocalDateTime LocalDateTime::operator-(const Duration &dur) const { } inline Duration LocalDateTime::operator-(const LocalDateTime &other) const { - auto mgp_difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3353,14 +3353,14 @@ inline Duration LocalDateTime::operator-(const LocalDateTime &other) const { } inline bool LocalDateTime::operator<(const LocalDateTime &other) const { - auto difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(local_date_time_diff, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string LocalDateTime::ToString() const { +inline std::string LocalDateTime::ToString() const { return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()) + "T" + std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + std::to_string(Millisecond()) + std::to_string(Microsecond()); @@ -3424,7 +3424,7 @@ inline int64_t Duration::Microseconds() const { return mgp::duration_get_microse inline bool Duration::operator==(const Duration &other) const { return util::DurationsEqual(ptr_, other.ptr_); } inline Duration Duration::operator+(const Duration &other) const { - auto mgp_sum = mgp::MemHandlerCallback(duration_add, ptr_, other.ptr_); + auto *mgp_sum = mgp::MemHandlerCallback(duration_add, ptr_, other.ptr_); auto sum = Duration(mgp_sum); mgp::duration_destroy(mgp_sum); @@ -3432,7 +3432,7 @@ inline Duration Duration::operator+(const Duration &other) const { } inline Duration Duration::operator-(const Duration &other) const { - auto mgp_difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); + auto *mgp_difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); auto difference = Duration(mgp_difference); mgp::duration_destroy(mgp_difference); @@ -3440,7 +3440,7 @@ inline Duration Duration::operator-(const Duration &other) const { } inline Duration Duration::operator-() const { - auto mgp_neg = mgp::MemHandlerCallback(duration_neg, ptr_); + auto *mgp_neg = mgp::MemHandlerCallback(duration_neg, ptr_); auto neg = Duration(mgp_neg); mgp::duration_destroy(mgp_neg); @@ -3448,14 +3448,14 @@ inline Duration Duration::operator-() const { } inline bool Duration::operator<(const Duration &other) const { - auto difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); + auto *difference = mgp::MemHandlerCallback(duration_sub, ptr_, other.ptr_); auto is_less = (mgp::duration_get_microseconds(difference) < 0); mgp::duration_destroy(difference); return is_less; } -inline const std::string Duration::ToString() const { return std::to_string(Microseconds()) + "ms"; } +inline std::string Duration::ToString() const { return std::to_string(Microseconds()) + "ms"; } /* #endregion */ @@ -3662,7 +3662,7 @@ inline List Value::ValueList() { return List(mgp::value_get_list(ptr_)); } -inline const Map Value::ValueMap() const { +inline Map Value::ValueMap() const { if (Type() != Type::Map) { throw ValueException("Type of value is wrong: expected Map."); } @@ -3675,7 +3675,7 @@ inline Map Value::ValueMap() { return Map(mgp::value_get_map(ptr_)); } -inline const Node Value::ValueNode() const { +inline Node Value::ValueNode() const { if (Type() != Type::Node) { throw ValueException("Type of value is wrong: expected Node."); } @@ -3688,7 +3688,7 @@ inline Node Value::ValueNode() { return Node(mgp::value_get_vertex(ptr_)); } -inline const Relationship Value::ValueRelationship() const { +inline Relationship Value::ValueRelationship() const { if (Type() != Type::Relationship) { throw ValueException("Type of value is wrong: expected Relationship."); } @@ -3701,7 +3701,7 @@ inline Relationship Value::ValueRelationship() { return Relationship(mgp::value_get_edge(ptr_)); } -inline const Path Value::ValuePath() const { +inline Path Value::ValuePath() const { if (Type() != Type::Path) { throw ValueException("Type of value is wrong: expected Path."); } @@ -3714,7 +3714,7 @@ inline Path Value::ValuePath() { return Path(mgp::value_get_path(ptr_)); } -inline const Date Value::ValueDate() const { +inline Date Value::ValueDate() const { if (Type() != Type::Date) { throw ValueException("Type of value is wrong: expected Date."); } @@ -3727,7 +3727,7 @@ inline Date Value::ValueDate() { return Date(mgp::value_get_date(ptr_)); } -inline const LocalTime Value::ValueLocalTime() const { +inline LocalTime Value::ValueLocalTime() const { if (Type() != Type::LocalTime) { throw ValueException("Type of value is wrong: expected LocalTime."); } @@ -3740,7 +3740,7 @@ inline LocalTime Value::ValueLocalTime() { return LocalTime(mgp::value_get_local_time(ptr_)); } -inline const LocalDateTime Value::ValueLocalDateTime() const { +inline LocalDateTime Value::ValueLocalDateTime() const { if (Type() != Type::LocalDateTime) { throw ValueException("Type of value is wrong: expected LocalDateTime."); } @@ -3753,7 +3753,7 @@ inline LocalDateTime Value::ValueLocalDateTime() { return LocalDateTime(mgp::value_get_local_date_time(ptr_)); } -inline const Duration Value::ValueDuration() const { +inline Duration Value::ValueDuration() const { if (Type() != Type::Duration) { throw ValueException("Type of value is wrong: expected Duration."); } @@ -3921,7 +3921,7 @@ inline std::ostream &operator<<(std::ostream &os, const mgp::Type &type) { } } -inline const std::string Value::ToString() const { +inline std::string Value::ToString() const { const mgp::Type &type = Type(); switch (type) { case Type::Null: @@ -3965,85 +3965,85 @@ inline const std::string Value::ToString() const { inline Record::Record(mgp_result_record *record) : record_(record) {} inline void Record::Insert(const char *field_name, bool value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_bool, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_bool, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, std::int64_t value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_int, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_int, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, double value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_double, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_double, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, std::string_view value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const char *value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const List &list) { - auto mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); + auto *mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Map &map) { - auto mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); + auto *mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Node &node) { - auto mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); + auto *mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Relationship &relationship) { - auto mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); + auto *mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Path &path) { - auto mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); + auto *mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Date &date) { - auto mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); + auto *mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const LocalTime &local_time) { - auto mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); + auto *mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const LocalDateTime &local_date_time) { - auto mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); + auto *mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } inline void Record::Insert(const char *field_name, const Duration &duration) { - auto mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); + auto *mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); { mgp::result_record_insert(record_, field_name, mgp_val); } mgp::value_destroy(mgp_val); } @@ -4086,8 +4086,8 @@ inline void Record::Insert(const char *field_name, const Value &value) { inline RecordFactory::RecordFactory(mgp_result *result) : result_(result) {} -inline const Record RecordFactory::NewRecord() const { - auto record = mgp::result_new_record(result_); +inline Record RecordFactory::NewRecord() const { + auto *record = mgp::result_new_record(result_); if (record == nullptr) { throw mg_exception::NotEnoughMemoryException(); } @@ -4107,85 +4107,85 @@ inline void RecordFactory::SetErrorMessage(const char *error_msg) const { inline Result::Result(mgp_func_result *result) : result_(result) {} inline void Result::SetValue(bool value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_bool, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_bool, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(std::int64_t value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_int, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_int, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(double value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_double, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_double, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(std::string_view value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value.data()); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const char *value) { - auto mgp_val = mgp::MemHandlerCallback(value_make_string, value); + auto *mgp_val = mgp::MemHandlerCallback(value_make_string, value); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const List &list) { - auto mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); + auto *mgp_val = mgp::value_make_list(mgp::MemHandlerCallback(list_copy, list.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Map &map) { - auto mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); + auto *mgp_val = mgp::value_make_map(mgp::MemHandlerCallback(map_copy, map.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Node &node) { - auto mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); + auto *mgp_val = mgp::value_make_vertex(mgp::MemHandlerCallback(vertex_copy, node.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Relationship &relationship) { - auto mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); + auto *mgp_val = mgp::value_make_edge(mgp::MemHandlerCallback(edge_copy, relationship.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Path &path) { - auto mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); + auto *mgp_val = mgp::value_make_path(mgp::MemHandlerCallback(path_copy, path.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Date &date) { - auto mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); + auto *mgp_val = mgp::value_make_date(mgp::MemHandlerCallback(date_copy, date.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const LocalTime &local_time) { - auto mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); + auto *mgp_val = mgp::value_make_local_time(mgp::MemHandlerCallback(local_time_copy, local_time.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const LocalDateTime &local_date_time) { - auto mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); + auto *mgp_val = mgp::value_make_local_date_time(mgp::MemHandlerCallback(local_date_time_copy, local_date_time.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } inline void Result::SetValue(const Duration &duration) { - auto mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); + auto *mgp_val = mgp::value_make_duration(mgp::MemHandlerCallback(duration_copy, duration.ptr_)); { mgp::MemHandlerCallback(func_result_set_value, result_, mgp_val); } mgp::value_destroy(mgp_val); } @@ -4222,7 +4222,7 @@ inline Parameter::Parameter(std::string_view name, Type type, const char *defaul : name(name), type_(type), optional(true), default_value(Value(default_value)) {} inline Parameter::Parameter(std::string_view name, Type type, Value default_value) - : name(name), type_(type), optional(true), default_value(default_value) {} + : name(name), type_(type), optional(true), default_value(std::move(default_value)) {} inline Parameter::Parameter(std::string_view name, std::pair list_type) : name(name), type_(list_type.first), list_item_type_(list_type.second) {} @@ -4232,7 +4232,7 @@ inline Parameter::Parameter(std::string_view name, std::pair list_ty type_(list_type.first), list_item_type_(list_type.second), optional(true), - default_value(default_value) {} + default_value(std::move(default_value)) {} inline mgp_type *Parameter::GetMGPType() const { if (type_ == Type::List) { diff --git a/query_modules/schema.cpp b/query_modules/schema.cpp index d5a657e98..1b3035bab 100644 --- a/query_modules/schema.cpp +++ b/query_modules/schema.cpp @@ -111,7 +111,6 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) { mgp::MemoryDispatcherGuard guard{memory}; - ; const auto record_factory = mgp::RecordFactory(result); try { const mgp::Graph graph = mgp::Graph(memgraph_graph); @@ -570,7 +569,6 @@ void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *resul extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) { try { mgp::MemoryDispatcherGuard guard{memory}; - ; AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {}, {mgp::Return(Schema::kReturnNodeType, mgp::Type::String), diff --git a/src/audit/log.cpp b/src/audit/log.cpp index 635898a16..4222ebb28 100644 --- a/src/audit/log.cpp +++ b/src/audit/log.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "storage/v2/temporal.hpp" #include "utils/logging.hpp" @@ -87,8 +88,8 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) { return ret; } -Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis) - : storage_directory_(storage_directory), +Log::Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis) + : storage_directory_(std::move(storage_directory)), buffer_size_(buffer_size), buffer_flush_interval_millis_(buffer_flush_interval_millis), started_(false) {} diff --git a/src/audit/log.hpp b/src/audit/log.hpp index 8def3ede5..accf41fe8 100644 --- a/src/audit/log.hpp +++ b/src/audit/log.hpp @@ -36,7 +36,7 @@ class Log { }; public: - Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis); + Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis); ~Log(); diff --git a/src/auth/models.cpp b/src/auth/models.cpp index 263131df3..5415dc08d 100644 --- a/src/auth/models.cpp +++ b/src/auth/models.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -560,20 +561,20 @@ Databases Databases::Deserialize(const nlohmann::json &data) { } #endif -User::User() {} +User::User() = default; User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {} -User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions) - : username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {} +User::User(const std::string &username, std::string password_hash, const Permissions &permissions) + : username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {} #ifdef MG_ENTERPRISE -User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions, +User::User(const std::string &username, std::string password_hash, const Permissions &permissions, FineGrainedAccessHandler fine_grained_access_handler, Databases db_access) : username_(utils::ToLowerCase(username)), - password_hash_(password_hash), + password_hash_(std::move(password_hash)), permissions_(permissions), fine_grained_access_handler_(std::move(fine_grained_access_handler)), - database_access_(db_access) {} + database_access_(std::move(db_access)) {} #endif bool User::CheckPassword(const std::string &password) { diff --git a/src/auth/models.hpp b/src/auth/models.hpp index 33ba28f80..9f66d3119 100644 --- a/src/auth/models.hpp +++ b/src/auth/models.hpp @@ -14,6 +14,7 @@ #include #include +#include #include "dbms/constants.hpp" #include "utils/logging.hpp" @@ -301,8 +302,8 @@ class Databases final { bool Contains(const std::string &db) const; bool GetAllowAll() const { return allow_all_; } - const std::set &GetGrants() const { return grants_dbs_; } - const std::set &GetDenies() const { return denies_dbs_; } + const std::set> &GetGrants() const { return grants_dbs_; } + const std::set> &GetDenies() const { return denies_dbs_; } const std::string &GetDefault() const; nlohmann::json Serialize() const; @@ -310,14 +311,17 @@ class Databases final { static Databases Deserialize(const nlohmann::json &data); private: - Databases(bool allow_all, std::set grant, std::set deny, - const std::string &default_db = dbms::kDefaultDB) - : grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {} + Databases(bool allow_all, std::set> grant, std::set> deny, + std::string default_db = dbms::kDefaultDB) + : grants_dbs_(std::move(grant)), + denies_dbs_(std::move(deny)), + allow_all_(allow_all), + default_db_(std::move(default_db)) {} - std::set grants_dbs_; //!< set of databases with granted access - std::set denies_dbs_; //!< set of databases with denied access - bool allow_all_; //!< flag to allow access to everything (denied overrides this) - std::string default_db_; //!< user's default database + std::set> grants_dbs_; //!< set of databases with granted access + std::set> denies_dbs_; //!< set of databases with denied access + bool allow_all_; //!< flag to allow access to everything (denied overrides this) + std::string default_db_; //!< user's default database }; #endif @@ -327,9 +331,9 @@ class User final { User(); explicit User(const std::string &username); - User(const std::string &username, const std::string &password_hash, const Permissions &permissions); + User(const std::string &username, std::string password_hash, const Permissions &permissions); #ifdef MG_ENTERPRISE - User(const std::string &username, const std::string &password_hash, const Permissions &permissions, + User(const std::string &username, std::string password_hash, const Permissions &permissions, FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {}); #endif User(const User &) = default; diff --git a/src/communication/bolt/client.hpp b/src/communication/bolt/client.hpp index 35be997d0..008fca907 100644 --- a/src/communication/bolt/client.hpp +++ b/src/communication/bolt/client.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "communication/bolt/v1/codes.hpp" @@ -34,8 +35,8 @@ class FailureResponseException : public utils::BasicException { explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {} - FailureResponseException(const std::string &code, const std::string &message) - : utils::BasicException{message}, code_{code} {} + FailureResponseException(std::string code, const std::string &message) + : utils::BasicException{message}, code_{std::move(code)} {} const std::string &code() const { return code_; } SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException) diff --git a/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp b/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp index 0d096352b..e82019d09 100644 --- a/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp +++ b/src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -51,7 +51,7 @@ enum class ChunkState : uint8_t { template class ChunkedDecoderBuffer { public: - ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); } + explicit ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); } /** * Reads data from the internal buffer. diff --git a/src/communication/bolt/v1/decoder/decoder.hpp b/src/communication/bolt/v1/decoder/decoder.hpp index 593f514f5..8a8821708 100644 --- a/src/communication/bolt/v1/decoder/decoder.hpp +++ b/src/communication/bolt/v1/decoder/decoder.hpp @@ -401,11 +401,11 @@ class Decoder { } auto &labels = dv.ValueList(); vertex.labels.reserve(labels.size()); - for (size_t i = 0; i < labels.size(); ++i) { - if (labels[i].type() != Value::Type::String) { + for (auto &label : labels) { + if (label.type() != Value::Type::String) { return false; } - vertex.labels.emplace_back(std::move(labels[i].ValueString())); + vertex.labels.emplace_back(std::move(label.ValueString())); } // read properties diff --git a/src/communication/bolt/v1/encoder/base_encoder.hpp b/src/communication/bolt/v1/encoder/base_encoder.hpp index ee34ec085..8ce802b82 100644 --- a/src/communication/bolt/v1/encoder/base_encoder.hpp +++ b/src/communication/bolt/v1/encoder/base_encoder.hpp @@ -111,12 +111,12 @@ class BaseEncoder { void WriteList(const std::vector &value) { WriteTypeSize(value.size(), MarkerList); - for (auto &x : value) WriteValue(x); + for (const auto &x : value) WriteValue(x); } void WriteMap(const std::map &value) { WriteTypeSize(value.size(), MarkerMap); - for (auto &x : value) { + for (const auto &x : value) { WriteString(x.first); WriteValue(x.second); } @@ -205,11 +205,11 @@ class BaseEncoder { WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3); WriteRAW(utils::UnderlyingCast(Signature::Path)); WriteTypeSize(path.vertices.size(), MarkerList); - for (auto &v : path.vertices) WriteVertex(v); + for (const auto &v : path.vertices) WriteVertex(v); WriteTypeSize(path.edges.size(), MarkerList); - for (auto &e : path.edges) WriteEdge(e); + for (const auto &e : path.edges) WriteEdge(e); WriteTypeSize(path.indices.size(), MarkerList); - for (auto &i : path.indices) WriteInt(i); + for (const auto &i : path.indices) WriteInt(i); } void WriteDate(const utils::Date &date) { diff --git a/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp b/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp index 6751ce48b..9e481d0f7 100644 --- a/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp +++ b/src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -48,7 +48,7 @@ namespace memgraph::communication::bolt { template class ChunkedEncoderBuffer { public: - ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {} + explicit ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {} /** * Writes n values into the buffer. If n is bigger than whole chunk size diff --git a/src/communication/bolt/v1/encoder/client_encoder.hpp b/src/communication/bolt/v1/encoder/client_encoder.hpp index 49f3f0c16..34a5a9b50 100644 --- a/src/communication/bolt/v1/encoder/client_encoder.hpp +++ b/src/communication/bolt/v1/encoder/client_encoder.hpp @@ -39,7 +39,7 @@ class ClientEncoder : private BaseEncoder { using BaseEncoder::buffer_; public: - ClientEncoder(Buffer &buffer) : BaseEncoder(buffer) {} + explicit ClientEncoder(Buffer &buffer) : BaseEncoder(buffer) {} using BaseEncoder::UpdateVersion; diff --git a/src/communication/bolt/v1/encoder/encoder.hpp b/src/communication/bolt/v1/encoder/encoder.hpp index bc3203d90..028c4bf37 100644 --- a/src/communication/bolt/v1/encoder/encoder.hpp +++ b/src/communication/bolt/v1/encoder/encoder.hpp @@ -32,7 +32,7 @@ class Encoder : private BaseEncoder { using BaseEncoder::buffer_; public: - Encoder(Buffer &buffer) : BaseEncoder(buffer) {} + explicit Encoder(Buffer &buffer) : BaseEncoder(buffer) {} using BaseEncoder::UpdateVersion; diff --git a/src/communication/bolt/v1/states/handlers.hpp b/src/communication/bolt/v1/states/handlers.hpp index 26c995719..f873f0e6e 100644 --- a/src/communication/bolt/v1/states/handlers.hpp +++ b/src/communication/bolt/v1/states/handlers.hpp @@ -93,7 +93,7 @@ State HandlePullDiscard(TSession &session, std::optional n, std::optional AuthenticateUser(TSession &session, Value &metadata) { std::string username; std::string password; if (data["scheme"].ValueString() == "basic") { - if (!data.count("principal")) { // Special case principal = "" + if (!data.contains("principal")) { // Special case principal = "" spdlog::warn("The client didn't supply the principal field! Trying with \"\"..."); data["principal"] = ""; } - if (!data.count("credentials")) { // Special case credentials = "" + if (!data.contains("credentials")) { // Special case credentials = "" spdlog::warn("The client didn't supply the credentials field! Trying with \"\"..."); data["credentials"] = ""; } @@ -118,7 +118,7 @@ std::optional GetMetadataV4(TSession &session, const Marker marker) { } auto &data = metadata.ValueMap(); - if (!data.count("user_agent")) { + if (!data.contains("user_agent")) { spdlog::warn("The client didn't supply the user agent!"); return std::nullopt; } @@ -142,7 +142,7 @@ std::optional GetInitDataV5(TSession &session, const Marker marker) { } const auto &data = metadata.ValueMap(); - if (!data.count("user_agent")) { + if (!data.contains("user_agent")) { spdlog::warn("The client didn't supply the user agent!"); return std::nullopt; } diff --git a/src/communication/bolt/v1/value.hpp b/src/communication/bolt/v1/value.hpp index f17e4e2a6..9f189b1f7 100644 --- a/src/communication/bolt/v1/value.hpp +++ b/src/communication/bolt/v1/value.hpp @@ -91,7 +91,7 @@ struct UnboundedEdge { * The decoder writes data into this structure. */ struct Path { - Path() {} + Path() = default; Path(const std::vector &vertices, const std::vector &edges) { // Helper function. Looks for the given element in the collection. If found, diff --git a/src/communication/client.hpp b/src/communication/client.hpp index c0f7cfad8..64507a321 100644 --- a/src/communication/client.hpp +++ b/src/communication/client.hpp @@ -132,7 +132,7 @@ class Client final { */ class ClientInputStream final { public: - ClientInputStream(Client &client); + explicit ClientInputStream(Client &client); ClientInputStream(const ClientInputStream &) = delete; ClientInputStream(ClientInputStream &&) = delete; @@ -156,7 +156,7 @@ class ClientInputStream final { */ class ClientOutputStream final { public: - ClientOutputStream(Client &client); + explicit ClientOutputStream(Client &client); ClientOutputStream(const ClientOutputStream &) = delete; ClientOutputStream(ClientOutputStream &&) = delete; diff --git a/src/communication/context.cpp b/src/communication/context.cpp index 53cb4586b..18794a4cd 100644 --- a/src/communication/context.cpp +++ b/src/communication/context.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -34,7 +34,7 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) { } ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) { - if (key_file != "" && cert_file != "") { + if (!key_file.empty() && !cert_file.empty()) { MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1, "Couldn't load client certificate from file: {}", cert_file); MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1, @@ -124,7 +124,7 @@ ServerContext &ServerContext::operator=(ServerContext &&other) noexcept { return *this; } -ServerContext::~ServerContext() {} +ServerContext::~ServerContext() = default; SSL_CTX *ServerContext::context() { MG_ASSERT(ctx_); diff --git a/src/communication/helpers.cpp b/src/communication/helpers.cpp index 0f91c0ff6..2374e110a 100644 --- a/src/communication/helpers.cpp +++ b/src/communication/helpers.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -15,7 +15,7 @@ namespace memgraph::communication { -const std::string SslGetLastError() { +std::string SslGetLastError() { char buff[2048]; auto err = ERR_get_error(); ERR_error_string_n(err, buff, sizeof(buff)); diff --git a/src/communication/helpers.hpp b/src/communication/helpers.hpp index b82b68a38..f50697162 100644 --- a/src/communication/helpers.hpp +++ b/src/communication/helpers.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -18,6 +18,6 @@ namespace memgraph::communication { /** * This function reads and returns a string describing the last OpenSSL error. */ -const std::string SslGetLastError(); +std::string SslGetLastError(); } // namespace memgraph::communication diff --git a/src/communication/http/listener.hpp b/src/communication/http/listener.hpp index 029bf5ca1..fac4cfaf3 100644 --- a/src/communication/http/listener.hpp +++ b/src/communication/http/listener.hpp @@ -38,7 +38,7 @@ class Listener final : public std::enable_shared_from_this static std::shared_ptr Create(Args &&...args) { diff --git a/src/communication/listener.hpp b/src/communication/listener.hpp index cbb6c0b2f..066d3c53c 100644 --- a/src/communication/listener.hpp +++ b/src/communication/listener.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -51,13 +52,13 @@ class Listener final { using SessionHandler = Session; public: - Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name, + Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, std::string service_name, size_t workers_count) : data_(data), alive_(false), context_(context), inactivity_timeout_sec_(inactivity_timeout_sec), - service_name_(service_name), + service_name_(std::move(service_name)), workers_count_(workers_count) {} ~Listener() { diff --git a/src/communication/result_stream_faker.hpp b/src/communication/result_stream_faker.hpp index 1c899b31b..f8786dd43 100644 --- a/src/communication/result_stream_faker.hpp +++ b/src/communication/result_stream_faker.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -80,7 +80,7 @@ class ResultStreamFaker { std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); }); // convert all the results into strings, and track max column width - auto &results_data = results.GetResults(); + const auto &results_data = results.GetResults(); std::vector> result_strings(results_data.size(), std::vector(column_widths.size())); for (int row_ind = 0; row_ind < static_cast(results_data.size()); ++row_ind) { diff --git a/src/communication/session.hpp b/src/communication/session.hpp index e44bf9532..a6c439ced 100644 --- a/src/communication/session.hpp +++ b/src/communication/session.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -51,7 +52,8 @@ using InputStream = Buffer::ReadEnd; */ class OutputStream final { public: - OutputStream(std::function write_function) : write_function_(write_function) {} + explicit OutputStream(std::function write_function) + : write_function_(std::move(write_function)) {} OutputStream(const OutputStream &) = delete; OutputStream(OutputStream &&) = delete; diff --git a/src/communication/v2/listener.hpp b/src/communication/v2/listener.hpp index 82d6fc2cb..1061f8c09 100644 --- a/src/communication/v2/listener.hpp +++ b/src/communication/v2/listener.hpp @@ -47,7 +47,7 @@ class Listener final : public std::enable_shared_from_this static std::shared_ptr Create(Args &&...args) { diff --git a/src/communication/v2/session.hpp b/src/communication/v2/session.hpp index 37e55e112..b54607729 100644 --- a/src/communication/v2/session.hpp +++ b/src/communication/v2/session.hpp @@ -76,7 +76,7 @@ using tcp = boost::asio::ip::tcp; class OutputStream final { public: explicit OutputStream(std::function write_function) - : write_function_(write_function) {} + : write_function_(std::move(write_function)) {} OutputStream(const OutputStream &) = delete; OutputStream(OutputStream &&) = delete; diff --git a/src/communication/websocket/server.hpp b/src/communication/websocket/server.hpp index 0853d3ebc..31a592932 100644 --- a/src/communication/websocket/server.hpp +++ b/src/communication/websocket/server.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -16,6 +16,7 @@ #include #include #include +#include #include "communication/websocket/listener.hpp" #include "io/network/endpoint.hpp" @@ -45,7 +46,7 @@ class Server final { class LoggingSink : public spdlog::sinks::base_sink { public: - explicit LoggingSink(std::weak_ptr listener) : listener_(listener) {} + explicit LoggingSink(std::weak_ptr listener) : listener_(std::move(listener)) {} private: void sink_it_(const spdlog::details::log_msg &msg) override; diff --git a/src/dbms/dbms_handler.hpp b/src/dbms/dbms_handler.hpp index 990420bf8..27ab963d4 100644 --- a/src/dbms/dbms_handler.hpp +++ b/src/dbms/dbms_handler.hpp @@ -542,7 +542,7 @@ class DbmsHandler { DatabaseHandler db_handler_; //!< multi-tenancy storage handler std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory - std::set defunct_dbs_; //!< Databases that are in an unknown state due to various failures + std::set> defunct_dbs_; //!< Databases that are in an unknown state due to various failures #else mutable utils::Gatekeeper db_gatekeeper_; //!< Single databases gatekeeper #endif diff --git a/src/dbms/handler.hpp b/src/dbms/handler.hpp index a7622e6b2..568b2fc7c 100644 --- a/src/dbms/handler.hpp +++ b/src/dbms/handler.hpp @@ -38,7 +38,7 @@ class Handler { * @brief Empty Handler constructor. * */ - Handler() {} + Handler() = default; /** * @brief Generate a new context and corresponding configuration. diff --git a/src/glue/SessionHL.cpp b/src/glue/SessionHL.cpp index a84f44974..cc7910d1c 100644 --- a/src/glue/SessionHL.cpp +++ b/src/glue/SessionHL.cpp @@ -10,6 +10,7 @@ // licenses/APL.txt. #include +#include #include "gflags/gflags.h" #include "audit/log.hpp" @@ -272,7 +273,7 @@ void SessionHL::Configure(const std::map *auth @@ -289,7 +290,7 @@ SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context, audit_log_(audit_log), #endif auth_(auth), - endpoint_(endpoint), + endpoint_(std::move(endpoint)), implicit_db_(dbms::kDefaultDB) { // Metrics update memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions); diff --git a/src/glue/SessionHL.hpp b/src/glue/SessionHL.hpp index cc53ae08a..374d2464e 100644 --- a/src/glue/SessionHL.hpp +++ b/src/glue/SessionHL.hpp @@ -23,7 +23,7 @@ class SessionHL final : public memgraph::communication::bolt::Session { public: SessionHL(memgraph::query::InterpreterContext *interpreter_context, - const memgraph::communication::v2::ServerEndpoint &endpoint, + memgraph::communication::v2::ServerEndpoint endpoint, memgraph::communication::v2::InputStream *input_stream, memgraph::communication::v2::OutputStream *output_stream, memgraph::utils::Synchronized *auth diff --git a/src/glue/communication.cpp b/src/glue/communication.cpp index 5db3e2726..fdf5129f6 100644 --- a/src/glue/communication.cpp +++ b/src/glue/communication.cpp @@ -202,7 +202,7 @@ storage::Result> ToBoltGraph(const query::Graph &gr for (const auto &v : graph.vertices()) { auto maybe_vertex = ToBoltVertex(v, db, view); if (maybe_vertex.HasError()) return maybe_vertex.GetError(); - vertices.emplace_back(Value(std::move(*maybe_vertex))); + vertices.emplace_back(std::move(*maybe_vertex)); } map.emplace("nodes", Value(vertices)); @@ -211,7 +211,7 @@ storage::Result> ToBoltGraph(const query::Graph &gr for (const auto &e : graph.edges()) { auto maybe_edge = ToBoltEdge(e, db, view); if (maybe_edge.HasError()) return maybe_edge.GetError(); - edges.emplace_back(Value(std::move(*maybe_edge))); + edges.emplace_back(std::move(*maybe_edge)); } map.emplace("edges", Value(edges)); diff --git a/src/helpers.hpp b/src/helpers.hpp index 1cf4e0ec0..67c1906e7 100644 --- a/src/helpers.hpp +++ b/src/helpers.hpp @@ -31,7 +31,7 @@ inline void LoadConfig(const std::string &product_name) { std::vector configs = {fs::path("/etc/memgraph/memgraph.conf")}; if (getenv("HOME") != nullptr) configs.emplace_back(fs::path(getenv("HOME")) / fs::path(".memgraph/config")); { - auto memgraph_config = getenv("MEMGRAPH_CONFIG"); + auto *memgraph_config = getenv("MEMGRAPH_CONFIG"); if (memgraph_config != nullptr) { auto path = fs::path(memgraph_config); MG_ASSERT(fs::exists(path), "MEMGRAPH_CONFIG environment variable set to nonexisting path: {}", diff --git a/src/http_handlers/metrics.hpp b/src/http_handlers/metrics.hpp index d4620b774..82131d311 100644 --- a/src/http_handlers/metrics.hpp +++ b/src/http_handlers/metrics.hpp @@ -23,7 +23,6 @@ #include #include #include "storage/v2/storage.hpp" -#include "utils/event_gauge.hpp" #include "utils/event_histogram.hpp" namespace memgraph::http { diff --git a/src/integrations/kafka/consumer.hpp b/src/integrations/kafka/consumer.hpp index 34e8b1bdb..5f698a4d1 100644 --- a/src/integrations/kafka/consumer.hpp +++ b/src/integrations/kafka/consumer.hpp @@ -171,10 +171,10 @@ class Consumer final : public RdKafka::EventCb { class ConsumerRebalanceCb : public RdKafka::RebalanceCb { public: - ConsumerRebalanceCb(std::string consumer_name); + explicit ConsumerRebalanceCb(std::string consumer_name); void rebalance_cb(RdKafka::KafkaConsumer *consumer, RdKafka::ErrorCode err, - std::vector &partitions) override final; + std::vector &partitions) final; void set_offset(int64_t offset); diff --git a/src/integrations/pulsar/consumer.cpp b/src/integrations/pulsar/consumer.cpp index b3a65050b..f004cf6dc 100644 --- a/src/integrations/pulsar/consumer.cpp +++ b/src/integrations/pulsar/consumer.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -67,7 +67,7 @@ utils::BasicResult> GetBatch(TConsumer &consum return std::move(batch); case pulsar_client::Result::ResultOk: if (message.getMessageId() != last_message_id) { - batch.emplace_back(Message{std::move(message)}); + batch.emplace_back(std::move(message)); } break; default: diff --git a/src/io/network/endpoint.hpp b/src/io/network/endpoint.hpp index e10c80657..281be2162 100644 --- a/src/io/network/endpoint.hpp +++ b/src/io/network/endpoint.hpp @@ -48,8 +48,8 @@ struct Endpoint { uint16_t port{0}; IpFamily family{IpFamily::NONE}; - static std::optional> ParseSocketOrAddress( - const std::string &address, const std::optional default_port); + static std::optional> ParseSocketOrAddress(const std::string &address, + std::optional default_port); /** * Tries to parse the given string as either a socket address or ip address. @@ -61,8 +61,8 @@ struct Endpoint { * it into an ip address and a port number; even if a default port is given, * it won't be used, as we expect that it is given in the address string. */ - static std::optional> ParseSocketOrIpAddress( - const std::string &address, const std::optional default_port); + static std::optional> ParseSocketOrIpAddress(const std::string &address, + std::optional default_port); /** * Tries to parse given string as either socket address or hostname. @@ -72,7 +72,7 @@ struct Endpoint { * After we parse hostname and port we try to resolve the hostname into an ip_address. */ static std::optional> ParseHostname(const std::string &address, - const std::optional default_port); + std::optional default_port); static IpFamily GetIpFamily(const std::string &address); diff --git a/src/io/network/epoll.hpp b/src/io/network/epoll.hpp index 09c485eb5..2f0d0dfd1 100644 --- a/src/io/network/epoll.hpp +++ b/src/io/network/epoll.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -32,7 +32,7 @@ class Epoll { public: using Event = struct epoll_event; - Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) { + explicit Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) { // epoll_create1 returns an error if there is a logical error in our code // (for example invalid flags) or if there is irrecoverable error. In both // cases it is best to terminate. diff --git a/src/io/network/socket.hpp b/src/io/network/socket.hpp index 22e787611..c2b6354af 100644 --- a/src/io/network/socket.hpp +++ b/src/io/network/socket.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "io/network/endpoint.hpp" @@ -201,7 +202,7 @@ class Socket { bool WaitForReadyWrite(); private: - Socket(int fd, const Endpoint &endpoint) : socket_(fd), endpoint_(endpoint) {} + Socket(int fd, Endpoint endpoint) : socket_(fd), endpoint_(std::move(endpoint)) {} int socket_ = -1; Endpoint endpoint_; diff --git a/src/kvstore/kvstore.cpp b/src/kvstore/kvstore.cpp index 383324238..877d6f9bd 100644 --- a/src/kvstore/kvstore.cpp +++ b/src/kvstore/kvstore.cpp @@ -128,7 +128,7 @@ KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, b KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); } -KVStore::iterator::~iterator() {} +KVStore::iterator::~iterator() = default; KVStore::iterator &KVStore::iterator::operator=(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); diff --git a/src/query/common.hpp b/src/query/common.hpp index 8f1b0a94c..6f45760fe 100644 --- a/src/query/common.hpp +++ b/src/query/common.hpp @@ -41,7 +41,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b); /// the define how respective elements compare. class TypedValueVectorCompare final { public: - TypedValueVectorCompare() {} + TypedValueVectorCompare() = default; explicit TypedValueVectorCompare(const std::vector &ordering) : ordering_(ordering) {} template @@ -147,8 +147,8 @@ concept AccessorWithUpdateProperties = requires(T accessor, /// /// @throw QueryRuntimeException if value cannot be set as a property value template -auto UpdatePropertiesChecked(T *record, std::map &properties) -> - typename std::remove_referenceUpdateProperties(properties).GetValue())>::type { +auto UpdatePropertiesChecked(T *record, std::map &properties) + -> std::remove_reference_tUpdateProperties(properties).GetValue())> { try { auto maybe_values = record->UpdateProperties(properties); if (maybe_values.HasError()) { diff --git a/src/query/cypher_query_interpreter.hpp b/src/query/cypher_query_interpreter.hpp index f33fa61e2..f1e9113a3 100644 --- a/src/query/cypher_query_interpreter.hpp +++ b/src/query/cypher_query_interpreter.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "query/config.hpp" #include "query/frontend/semantic/required_privileges.hpp" #include "query/frontend/semantic/symbol_generator.hpp" @@ -98,8 +100,8 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map root, double cost, AstStorage storage, - const SymbolTable &symbol_table) - : root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {} + SymbolTable symbol_table) + : root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(std::move(symbol_table)) {} const plan::LogicalOperator &GetRoot() const override { return *root_; } double GetCost() const override { return cost_; } diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index 75ec1e9ae..f616dc5a2 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -40,7 +40,6 @@ class EdgeAccessor final { public: storage::EdgeAccessor impl_; - public: explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {} bool IsVisible(storage::View view) const { return impl_.IsVisible(view); } @@ -108,7 +107,6 @@ class VertexAccessor final { static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); } - public: explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {} bool IsVisible(storage::View view) const { return impl_.IsVisible(view); } diff --git a/src/query/dump.cpp b/src/query/dump.cpp index dd99f0f63..a1421cbf9 100644 --- a/src/query/dump.cpp +++ b/src/query/dump.cpp @@ -159,7 +159,7 @@ void DumpProperties(std::ostream *os, query::DbAccessor *dba, *os << "{"; if (property_id) { *os << kInternalPropertyId << ": " << *property_id; - if (store.size() > 0) *os << ", "; + if (!store.empty()) *os << ", "; } utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) { os << EscapeName(dba->PropertyToName(kv.first)) << ": "; @@ -228,7 +228,7 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba, const query::EdgeAccesso throw query::QueryRuntimeException("Unexpected error when getting properties."); } } - if (maybe_props->size() > 0) { + if (!maybe_props->empty()) { *os << " "; DumpProperties(os, dba, *maybe_props); } diff --git a/src/query/frame_change.hpp b/src/query/frame_change.hpp index 1d9ebc70c..32fe1f36e 100644 --- a/src/query/frame_change.hpp +++ b/src/query/frame_change.hpp @@ -47,9 +47,9 @@ struct CachedValue { if (!maybe_list.IsList()) { return false; } - auto &list = maybe_list.ValueList(); + const auto &list = maybe_list.ValueList(); TypedValue::Hash hash{}; - for (auto &element : list) { + for (const auto &element : list) { const auto key = hash(element); auto &vector_values = cache_[key]; if (!IsValueInVec(vector_values, element)) { diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index 6b7fdb2c6..dc11c3887 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -23,9 +23,7 @@ #include "storage/v2/property_value.hpp" #include "utils/typeinfo.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { struct LabelIx { static const utils::TypeInfo kType; @@ -62,8 +60,8 @@ inline bool operator!=(const PropertyIx &a, const PropertyIx &b) { return !(a == inline bool operator==(const EdgeTypeIx &a, const EdgeTypeIx &b) { return a.ix == b.ix && a.name == b.name; } inline bool operator!=(const EdgeTypeIx &a, const EdgeTypeIx &b) { return !(a == b); } -} // namespace query -} // namespace memgraph +} // namespace memgraph::query + namespace std { template <> @@ -83,9 +81,7 @@ struct hash { } // namespace std -namespace memgraph { - -namespace query { +namespace memgraph::query { class Tree; @@ -3577,5 +3573,4 @@ class ShowDatabasesQuery : public memgraph::query::Query { } }; -} // namespace query -} // namespace memgraph +} // namespace memgraph::query diff --git a/src/query/frontend/ast/pretty_print.cpp b/src/query/frontend/ast/pretty_print.cpp index 8275f1cb5..ef45afd7d 100644 --- a/src/query/frontend/ast/pretty_print.cpp +++ b/src/query/frontend/ast/pretty_print.cpp @@ -105,7 +105,7 @@ void PrintObject(std::ostream *out, const std::map &map); template void PrintObject(std::ostream *out, const T &arg) { - static_assert(!std::is_convertible::value, + static_assert(!std::is_convertible_v, "This overload shouldn't be called with pointers convertible " "to Expression *. This means your other PrintObject overloads aren't " "being called for certain AST nodes when they should (or perhaps such " diff --git a/src/query/frontend/opencypher/parser.hpp b/src/query/frontend/opencypher/parser.hpp index 3a17d27a8..f25d6e5e9 100644 --- a/src/query/frontend/opencypher/parser.hpp +++ b/src/query/frontend/opencypher/parser.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -31,7 +31,7 @@ class Parser { * @param query incoming query that has to be compiled into query plan * the first step is to generate AST */ - Parser(const std::string query) : query_(std::move(query)) { + explicit Parser(const std::string query) : query_(std::move(query)) { parser_.removeErrorListeners(); parser_.addErrorListener(&error_listener_); tree_ = parser_.cypher(); diff --git a/src/query/frontend/semantic/symbol.hpp b/src/query/frontend/semantic/symbol.hpp index 5381cb48d..77557b6fe 100644 --- a/src/query/frontend/semantic/symbol.hpp +++ b/src/query/frontend/semantic/symbol.hpp @@ -12,12 +12,11 @@ #pragma once #include +#include #include "utils/typeinfo.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { class Symbol { public: @@ -34,9 +33,13 @@ class Symbol { return enum_string[static_cast(type)]; } - Symbol() {} - Symbol(const std::string &name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1) - : name_(name), position_(position), user_declared_(user_declared), type_(type), token_position_(token_position) {} + Symbol() = default; + Symbol(std::string name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1) + : name_(std::move(name)), + position_(position), + user_declared_(user_declared), + type_(type), + token_position_(token_position) {} bool operator==(const Symbol &other) const { return position_ == other.position_ && name_ == other.name_ && type_ == other.type_; @@ -57,8 +60,8 @@ class Symbol { int64_t token_position_{-1}; }; -} // namespace query -} // namespace memgraph +} // namespace memgraph::query + namespace std { template <> diff --git a/src/query/frontend/semantic/symbol_generator.hpp b/src/query/frontend/semantic/symbol_generator.hpp index c69d8729e..207bbddbd 100644 --- a/src/query/frontend/semantic/symbol_generator.hpp +++ b/src/query/frontend/semantic/symbol_generator.hpp @@ -183,7 +183,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor { /// If property lookup for one symbol is visited more times, it is better to fetch all properties class PropertyLookupEvaluationModeVisitor : public ExpressionVisitor { public: - explicit PropertyLookupEvaluationModeVisitor() {} + explicit PropertyLookupEvaluationModeVisitor() = default; using ExpressionVisitor::Visit; diff --git a/src/query/frontend/semantic/symbol_table.hpp b/src/query/frontend/semantic/symbol_table.hpp index 23b4965c5..0b521356c 100644 --- a/src/query/frontend/semantic/symbol_table.hpp +++ b/src/query/frontend/semantic/symbol_table.hpp @@ -22,7 +22,7 @@ namespace memgraph::query { class SymbolTable final { public: - SymbolTable() {} + SymbolTable() = default; const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY, int32_t token_position = -1) { MG_ASSERT(table_.size() <= std::numeric_limits::max(), diff --git a/src/query/frontend/stripped.cpp b/src/query/frontend/stripped.cpp index 56d29032c..9740cd463 100644 --- a/src/query/frontend/stripped.cpp +++ b/src/query/frontend/stripped.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "query/exceptions.hpp" @@ -32,7 +33,7 @@ namespace memgraph::query::frontend { using namespace lexer_constants; -StrippedQuery::StrippedQuery(const std::string &query) : original_(query) { +StrippedQuery::StrippedQuery(std::string query) : original_(std::move(query)) { enum class Token { UNMATCHED, KEYWORD, // Including true, false and null. @@ -255,29 +256,29 @@ std::string GetFirstUtf8Symbol(const char *_s) { // According to // https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe // this checks if casting from const char * to uint8_t is undefined behaviour. - static_assert(std::is_same::value, + static_assert(std::is_same_v, "This library requires std::uint8_t to be implemented as " "unsigned char."); const uint8_t *s = reinterpret_cast(_s); if ((*s >> 7) == 0x00) return std::string(_s, _s + 1); if ((*s >> 5) == 0x06) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 2); } if ((*s >> 4) == 0x0e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 3); } if ((*s >> 3) == 0x1e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s3 = s + 3; + const auto *s3 = s + 3; if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); return std::string(_s, _s + 4); } @@ -286,29 +287,29 @@ std::string GetFirstUtf8Symbol(const char *_s) { // Return codepoint of first utf8 symbol and its encoded length. std::pair GetFirstUtf8SymbolCodepoint(const char *_s) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "This library requires std::uint8_t to be implemented as " "unsigned char."); const uint8_t *s = reinterpret_cast(_s); if ((*s >> 7) == 0x00) return {*s & 0x7f, 1}; if ((*s >> 5) == 0x06) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2}; } if ((*s >> 4) == 0x0e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3}; } if ((*s >> 3) == 0x1e) { - auto *s1 = s + 1; + const auto *s1 = s + 1; if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s2 = s + 2; + const auto *s2 = s + 2; if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); - auto *s3 = s + 3; + const auto *s3 = s + 3; if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4}; } @@ -336,7 +337,7 @@ int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(o int StrippedQuery::MatchString(int start) const { if (original_[start] != '"' && original_[start] != '\'') return 0; char start_char = original_[start]; - for (auto *p = original_.data() + start + 1; *p; ++p) { + for (const auto *p = original_.data() + start + 1; *p; ++p) { if (*p == start_char) return p - (original_.data() + start) + 1; if (*p == '\\') { ++p; @@ -346,7 +347,7 @@ int StrippedQuery::MatchString(int start) const { continue; } else if (*p == 'U' || *p == 'u') { int cnt = 0; - auto *r = p + 1; + const auto *r = p + 1; while (isxdigit(*r) && cnt < 8) { ++cnt; ++r; diff --git a/src/query/frontend/stripped.hpp b/src/query/frontend/stripped.hpp index 006fb1dd6..a20e28474 100644 --- a/src/query/frontend/stripped.hpp +++ b/src/query/frontend/stripped.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -40,7 +40,7 @@ class StrippedQuery { * * @param query Input query. */ - explicit StrippedQuery(const std::string &query); + explicit StrippedQuery(std::string query); /** * Copy constructor is deleted because we don't want to make unnecessary diff --git a/src/query/frontend/stripped_lexer_constants.hpp b/src/query/frontend/stripped_lexer_constants.hpp index 9ad95e6f5..21a14ae83 100644 --- a/src/query/frontend/stripped_lexer_constants.hpp +++ b/src/query/frontend/stripped_lexer_constants.hpp @@ -17,8 +17,7 @@ #include #include -namespace memgraph::query { -namespace lexer_constants { +namespace memgraph::query::lexer_constants { namespace trie { @@ -33,7 +32,7 @@ inline int Noop(int x) { return x; } class Trie { public: - Trie() {} + Trie() = default; Trie(std::initializer_list l) { for (const auto &s : l) { Insert(s); @@ -2934,5 +2933,4 @@ const trie::Trie kSpecialTokens = {";", "\xEF\xB9\x98", // u8"\ufe58" "\xEF\xB9\xA3", // u8"\ufe63" "\xEF\xBC\x8D"}; // u8"\uff0d" -} // namespace lexer_constants -} // namespace memgraph::query +} // namespace memgraph::query::lexer_constants diff --git a/src/query/interpret/eval.hpp b/src/query/interpret/eval.hpp index 333a7b1fa..916082bb2 100644 --- a/src/query/interpret/eval.hpp +++ b/src/query/interpret/eval.hpp @@ -825,8 +825,8 @@ class ExpressionEvaluator : public ExpressionVisitor { throw QueryRuntimeException("'coalesce' requires at least one argument."); } - for (int64_t i = 0; i < exprs.size(); ++i) { - TypedValue val(exprs[i]->Accept(*this), ctx_->memory); + for (auto &expr : exprs) { + TypedValue val(expr->Accept(*this), ctx_->memory); if (!val.IsNull()) { return val; } diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 5aad0ff07..354f13dc3 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -797,34 +797,33 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & std::vector typed_replica; typed_replica.reserve(replica_nfields); - typed_replica.emplace_back(TypedValue(replica.name)); - typed_replica.emplace_back(TypedValue(replica.socket_address)); + typed_replica.emplace_back(replica.name); + typed_replica.emplace_back(replica.socket_address); switch (replica.sync_mode) { case ReplicationQuery::SyncMode::SYNC: - typed_replica.emplace_back(TypedValue("sync")); + typed_replica.emplace_back("sync"); break; case ReplicationQuery::SyncMode::ASYNC: - typed_replica.emplace_back(TypedValue("async")); + typed_replica.emplace_back("async"); break; } - typed_replica.emplace_back(TypedValue(static_cast(replica.current_timestamp_of_replica))); - typed_replica.emplace_back( - TypedValue(static_cast(replica.current_number_of_timestamp_behind_master))); + typed_replica.emplace_back(static_cast(replica.current_timestamp_of_replica)); + typed_replica.emplace_back(static_cast(replica.current_number_of_timestamp_behind_master)); switch (replica.state) { case ReplicationQuery::ReplicaState::READY: - typed_replica.emplace_back(TypedValue("ready")); + typed_replica.emplace_back("ready"); break; case ReplicationQuery::ReplicaState::REPLICATING: - typed_replica.emplace_back(TypedValue("replicating")); + typed_replica.emplace_back("replicating"); break; case ReplicationQuery::ReplicaState::RECOVERY: - typed_replica.emplace_back(TypedValue("recovery")); + typed_replica.emplace_back("recovery"); break; case ReplicationQuery::ReplicaState::INVALID: - typed_replica.emplace_back(TypedValue("invalid")); + typed_replica.emplace_back("invalid"); break; } @@ -1962,11 +1961,11 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat result.reserve(kComputeStatisticsNumResults); result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first)); - result.emplace_back(TypedValue()); + result.emplace_back(); result.emplace_back(static_cast(stat_entry.second.count)); - result.emplace_back(TypedValue()); - result.emplace_back(TypedValue()); - result.emplace_back(TypedValue()); + result.emplace_back(); + result.emplace_back(); + result.emplace_back(); result.emplace_back(stat_entry.second.avg_degree); results.push_back(std::move(result)); }); @@ -2883,7 +2882,7 @@ auto ShowTransactions(const std::unordered_set &interpreters, con metadata_tv.emplace(md.first, TypedValue(md.second)); } } - results.back().push_back(TypedValue(metadata_tv)); + results.back().emplace_back(metadata_tv); } } return results; diff --git a/src/query/interpreter.hpp b/src/query/interpreter.hpp index 66231059d..5cb73cb07 100644 --- a/src/query/interpreter.hpp +++ b/src/query/interpreter.hpp @@ -174,7 +174,7 @@ struct CurrentDB { class Interpreter final { public: - Interpreter(InterpreterContext *interpreter_context); + explicit Interpreter(InterpreterContext *interpreter_context); Interpreter(InterpreterContext *interpreter_context, memgraph::dbms::DatabaseAccess db); Interpreter(const Interpreter &) = delete; Interpreter &operator=(const Interpreter &) = delete; diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index b68810ad7..1c8d021c7 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -207,8 +207,8 @@ void Once::OnceCursor::Shutdown() {} void Once::OnceCursor::Reset() { did_pull_ = false; } -CreateNode::CreateNode(const std::shared_ptr &input, const NodeCreationInfo &node_info) - : input_(input ? input : std::make_shared()), node_info_(node_info) {} +CreateNode::CreateNode(const std::shared_ptr &input, NodeCreationInfo node_info) + : input_(input ? input : std::make_shared()), node_info_(std::move(node_info)) {} // Creates a vertex on this GraphDb. Returns a reference to vertex placed on the // frame. @@ -298,12 +298,12 @@ void CreateNode::CreateNodeCursor::Shutdown() { input_cursor_->Shutdown(); } void CreateNode::CreateNodeCursor::Reset() { input_cursor_->Reset(); } -CreateExpand::CreateExpand(const NodeCreationInfo &node_info, const EdgeCreationInfo &edge_info, +CreateExpand::CreateExpand(NodeCreationInfo node_info, EdgeCreationInfo edge_info, const std::shared_ptr &input, Symbol input_symbol, bool existing_node) - : node_info_(node_info), - edge_info_(edge_info), + : node_info_(std::move(node_info)), + edge_info_(std::move(edge_info)), input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), existing_node_(existing_node) {} ACCEPT_WITH_INPUT(CreateExpand) @@ -447,7 +447,7 @@ class ScanAllCursor : public Cursor { explicit ScanAllCursor(const ScanAll &self, Symbol output_symbol, UniqueCursorPtr input_cursor, storage::View view, TVerticesFun get_vertices, const char *op_name) : self_(self), - output_symbol_(output_symbol), + output_symbol_(std::move(output_symbol)), input_cursor_(std::move(input_cursor)), view_(view), get_vertices_(std::move(get_vertices)), @@ -518,7 +518,7 @@ class ScanAllCursor : public Cursor { }; ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::View view) - : input_(input ? input : std::make_shared()), output_symbol_(output_symbol), view_(view) {} + : input_(input ? input : std::make_shared()), output_symbol_(std::move(output_symbol)), view_(view) {} ACCEPT_WITH_INPUT(ScanAll) @@ -561,13 +561,13 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const { ScanAllByLabelPropertyRange::ScanAllByLabelPropertyRange(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, std::optional lower_bound, std::optional upper_bound, storage::View view) : ScanAll(input, output_symbol, view), label_(label), property_(property), - property_name_(property_name), + property_name_(std::move(property_name)), lower_bound_(lower_bound), upper_bound_(upper_bound) { MG_ASSERT(lower_bound_ || upper_bound_, "Only one bound can be left out"); @@ -623,12 +623,12 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, Expression *expression, storage::View view) : ScanAll(input, output_symbol, view), label_(label), property_(property), - property_name_(property_name), + property_name_(std::move(property_name)), expression_(expression) { DMG_ASSERT(expression, "Expression is not optional."); } @@ -655,8 +655,11 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, storage::PropertyId property, - const std::string &property_name, storage::View view) - : ScanAll(input, output_symbol, view), label_(label), property_(property), property_name_(property_name) {} + std::string property_name, storage::View view) + : ScanAll(input, output_symbol, view), + label_(label), + property_(property), + property_name_(std::move(property_name)) {} ACCEPT_WITH_INPUT(ScanAllByLabelProperty) @@ -728,7 +731,7 @@ Expand::Expand(const std::shared_ptr &input, Symbol input_symbo Symbol edge_symbol, EdgeAtom::Direction direction, const std::vector &edge_types, bool existing_node, storage::View view) : input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, view_(view) {} @@ -962,15 +965,15 @@ ExpandVariable::ExpandVariable(const std::shared_ptr &input, Sy ExpansionLambda filter_lambda, std::optional weight_lambda, std::optional total_weight) : input_(input ? input : std::make_shared()), - input_symbol_(input_symbol), + input_symbol_(std::move(input_symbol)), common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, type_(type), is_reverse_(is_reverse), lower_bound_(lower_bound), upper_bound_(upper_bound), - filter_lambda_(filter_lambda), - weight_lambda_(weight_lambda), - total_weight_(total_weight) { + filter_lambda_(std::move(filter_lambda)), + weight_lambda_(std::move(weight_lambda)), + total_weight_(std::move(total_weight)) { DMG_ASSERT(type_ == EdgeAtom::Type::DEPTH_FIRST || type_ == EdgeAtom::Type::BREADTH_FIRST || type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS, "ExpandVariable can only be used with breadth first, depth first, " @@ -1758,7 +1761,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { if (found_it != total_cost_.end() && (found_it->second.IsNull() || (found_it->second <= next_weight).ValueBool())) return; - pq_.push({next_weight, depth + 1, vertex, edge}); + pq_.emplace(next_weight, depth + 1, vertex, edge); }; // Populates the priority queue structure with expansions @@ -1810,7 +1813,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { total_cost_.clear(); yielded_vertices_.clear(); - pq_.push({TypedValue(), 0, vertex, std::nullopt}); + pq_.emplace(TypedValue(), 0, vertex, std::nullopt); // We are adding the starting vertex to the set of yielded vertices // because we don't want to yield paths that end with the starting // vertex. @@ -2023,7 +2026,7 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor { } DirectedEdge directed_edge = {edge, direction, next_weight}; - pq_.push({next_weight, depth + 1, next_vertex, directed_edge}); + pq_.emplace(next_weight, depth + 1, next_vertex, directed_edge); }; // Populates the priority queue structure with expansions @@ -2314,8 +2317,8 @@ UniqueCursorPtr ExpandVariable::MakeCursor(utils::MemoryResource *mem) const { class ConstructNamedPathCursor : public Cursor { public: - ConstructNamedPathCursor(const ConstructNamedPath &self, utils::MemoryResource *mem) - : self_(self), input_cursor_(self_.input()->MakeCursor(mem)) {} + ConstructNamedPathCursor(ConstructNamedPath self, utils::MemoryResource *mem) + : self_(std::move(self)), input_cursor_(self_.input()->MakeCursor(mem)) {} bool Pull(Frame &frame, ExecutionContext &context) override { OOMExceptionEnabler oom_exception; @@ -2413,11 +2416,11 @@ Filter::Filter(const std::shared_ptr &input, Filter::Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression, - const Filters &all_filters) + Filters all_filters) : input_(input ? input : std::make_shared()), pattern_filters_(pattern_filters), expression_(expression), - all_filters_(all_filters) {} + all_filters_(std::move(all_filters)) {} bool Filter::Accept(HierarchicalLogicalOperatorVisitor &visitor) { if (visitor.PreVisit(*this)) { @@ -2478,7 +2481,7 @@ void Filter::FilterCursor::Shutdown() { input_cursor_->Shutdown(); } void Filter::FilterCursor::Reset() { input_cursor_->Reset(); } EvaluatePatternFilter::EvaluatePatternFilter(const std::shared_ptr &input, Symbol output_symbol) - : input_(input), output_symbol_(output_symbol) {} + : input_(input), output_symbol_(std::move(output_symbol)) {} ACCEPT_WITH_INPUT(EvaluatePatternFilter); @@ -2801,7 +2804,7 @@ void SetProperty::SetPropertyCursor::Shutdown() { input_cursor_->Shutdown(); } void SetProperty::SetPropertyCursor::Reset() { input_cursor_->Reset(); } SetProperties::SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op) - : input_(input), input_symbol_(input_symbol), rhs_(rhs), op_(op) {} + : input_(input), input_symbol_(std::move(input_symbol)), rhs_(rhs), op_(op) {} ACCEPT_WITH_INPUT(SetProperties) @@ -3000,7 +3003,7 @@ void SetProperties::SetPropertiesCursor::Reset() { input_cursor_->Reset(); } SetLabels::SetLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels) - : input_(input), input_symbol_(input_symbol), labels_(labels) {} + : input_(input), input_symbol_(std::move(input_symbol)), labels_(labels) {} ACCEPT_WITH_INPUT(SetLabels) @@ -3160,7 +3163,7 @@ void RemoveProperty::RemovePropertyCursor::Reset() { input_cursor_->Reset(); } RemoveLabels::RemoveLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels) - : input_(input), input_symbol_(input_symbol), labels_(labels) {} + : input_(input), input_symbol_(std::move(input_symbol)), labels_(labels) {} ACCEPT_WITH_INPUT(RemoveLabels) @@ -3234,7 +3237,7 @@ void RemoveLabels::RemoveLabelsCursor::Reset() { input_cursor_->Reset(); } EdgeUniquenessFilter::EdgeUniquenessFilter(const std::shared_ptr &input, Symbol expand_symbol, const std::vector &previous_symbols) - : input_(input), expand_symbol_(expand_symbol), previous_symbols_(previous_symbols) {} + : input_(input), expand_symbol_(std::move(expand_symbol)), previous_symbols_(previous_symbols) {} ACCEPT_WITH_INPUT(EdgeUniquenessFilter) @@ -4204,7 +4207,7 @@ void Optional::OptionalCursor::Reset() { Unwind::Unwind(const std::shared_ptr &input, Expression *input_expression, Symbol output_symbol) : input_(input ? input : std::make_shared()), input_expression_(input_expression), - output_symbol_(output_symbol) {} + output_symbol_(std::move(output_symbol)) {} ACCEPT_WITH_INPUT(Unwind) @@ -4535,7 +4538,7 @@ WITHOUT_SINGLE_INPUT(OutputTable); class OutputTableCursor : public Cursor { public: - OutputTableCursor(const OutputTable &self) : self_(self) {} + explicit OutputTableCursor(const OutputTable &self) : self_(self) {} bool Pull(Frame &frame, ExecutionContext &context) override { OOMExceptionEnabler oom_exception; @@ -4626,10 +4629,10 @@ CallProcedure::CallProcedure(std::shared_ptr input, std::string std::vector fields, std::vector symbols, Expression *memory_limit, size_t memory_scale, bool is_write, int64_t procedure_id, bool void_procedure) : input_(input ? input : std::make_shared()), - procedure_name_(name), - arguments_(args), - result_fields_(fields), - result_symbols_(symbols), + procedure_name_(std::move(name)), + arguments_(std::move(args)), + result_fields_(std::move(fields)), + result_symbols_(std::move(symbols)), memory_limit_(memory_limit), memory_scale_(memory_scale), is_write_(is_write), @@ -4983,7 +4986,7 @@ LoadCsv::LoadCsv(std::shared_ptr input, Expression *file, bool delimiter_(delimiter), quote_(quote), nullif_(nullif), - row_var_(row_var) { + row_var_(std::move(row_var)) { MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__); } @@ -5197,7 +5200,7 @@ Foreach::Foreach(std::shared_ptr input, std::shared_ptr()), update_clauses_(std::move(updates)), expression_(expr), - loop_variable_symbol_(loop_variable_symbol) {} + loop_variable_symbol_(std::move(loop_variable_symbol)) {} UniqueCursorPtr Foreach::MakeCursor(utils::MemoryResource *mem) const { memgraph::metrics::IncrementCounter(memgraph::metrics::ForeachOperator); diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index 7bb971752..03df07378 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -32,9 +32,7 @@ #include "utils/synchronized.hpp" #include "utils/visitor.hpp" -namespace memgraph { - -namespace query { +namespace memgraph::query { struct ExecutionContext; class ExpressionEvaluator; @@ -68,7 +66,7 @@ class Cursor { /// Perform cleanup which may throw an exception virtual void Shutdown() = 0; - virtual ~Cursor() {} + virtual ~Cursor() = default; }; /// unique_ptr to Cursor managed with a custom deleter. @@ -172,7 +170,7 @@ class LogicalOperator : public utils::Visitable &input, const NodeCreationInfo &node_info); + CreateNode(const std::shared_ptr &input, NodeCreationInfo node_info); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -445,7 +443,7 @@ class CreateExpand : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - CreateExpand() {} + CreateExpand() = default; /** @brief Construct @c CreateExpand. * @@ -459,8 +457,8 @@ class CreateExpand : public memgraph::query::plan::LogicalOperator { * @param existing_node @c bool indicating whether the @c node_atom refers to * an existing node. If @c false, the operator will also create the node. */ - CreateExpand(const NodeCreationInfo &node_info, const EdgeCreationInfo &edge_info, - const std::shared_ptr &input, Symbol input_symbol, bool existing_node); + CreateExpand(NodeCreationInfo node_info, EdgeCreationInfo edge_info, const std::shared_ptr &input, + Symbol input_symbol, bool existing_node); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -529,7 +527,7 @@ class ScanAll : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAll() {} + ScanAll() = default; ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -571,7 +569,7 @@ class ScanAllByLabel : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabel() {} + ScanAllByLabel() = default; ScanAllByLabel(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -606,7 +604,7 @@ class ScanAllByLabelPropertyRange : public memgraph::query::plan::ScanAll { /** Bound with expression which when evaluated produces the bound value. */ using Bound = utils::Bound; - ScanAllByLabelPropertyRange() {} + ScanAllByLabelPropertyRange() = default; /** * Constructs the operator for given label and property value in range * (inclusive). @@ -622,7 +620,7 @@ class ScanAllByLabelPropertyRange : public memgraph::query::plan::ScanAll { * @param view storage::View used when obtaining vertices. */ ScanAllByLabelPropertyRange(const std::shared_ptr &input, Symbol output_symbol, - storage::LabelId label, storage::PropertyId property, const std::string &property_name, + storage::LabelId label, storage::PropertyId property, std::string property_name, std::optional lower_bound, std::optional upper_bound, storage::View view = storage::View::OLD); @@ -675,7 +673,7 @@ class ScanAllByLabelPropertyValue : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabelPropertyValue() {} + ScanAllByLabelPropertyValue() = default; /** * Constructs the operator for given label and property value. * @@ -687,7 +685,7 @@ class ScanAllByLabelPropertyValue : public memgraph::query::plan::ScanAll { * @param view storage::View used when obtaining vertices. */ ScanAllByLabelPropertyValue(const std::shared_ptr &input, Symbol output_symbol, - storage::LabelId label, storage::PropertyId property, const std::string &property_name, + storage::LabelId label, storage::PropertyId property, std::string property_name, Expression *expression, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -727,9 +725,9 @@ class ScanAllByLabelProperty : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllByLabelProperty() {} + ScanAllByLabelProperty() = default; ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, storage::LabelId label, - storage::PropertyId property, const std::string &property_name, + storage::PropertyId property, std::string property_name, storage::View view = storage::View::OLD); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -763,7 +761,7 @@ class ScanAllById : public memgraph::query::plan::ScanAll { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ScanAllById() {} + ScanAllById() = default; ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, storage::View view = storage::View::OLD); @@ -842,7 +840,7 @@ class Expand : public memgraph::query::plan::LogicalOperator { EdgeAtom::Direction direction, const std::vector &edge_types, bool existing_node, storage::View view); - Expand() {} + Expand() = default; bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; @@ -950,7 +948,7 @@ class ExpandVariable : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ExpandVariable() {} + ExpandVariable() = default; /** * Creates a variable-length expansion. Most params are forwarded @@ -1073,10 +1071,10 @@ class ConstructNamedPath : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - ConstructNamedPath() {} + ConstructNamedPath() = default; ConstructNamedPath(const std::shared_ptr &input, Symbol path_symbol, const std::vector &path_elements) - : input_(input), path_symbol_(path_symbol), path_elements_(path_elements) {} + : input_(input), path_symbol_(std::move(path_symbol)), path_elements_(path_elements) {} bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -1108,13 +1106,13 @@ class Filter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Filter() {} + Filter() = default; Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression); Filter(const std::shared_ptr &input, const std::vector> &pattern_filters, Expression *expression, - const Filters &all_filters); + Filters all_filters); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector ModifiedSymbols(const SymbolTable &) const override; @@ -1126,12 +1124,12 @@ class Filter : public memgraph::query::plan::LogicalOperator { std::shared_ptr input_; std::vector> pattern_filters_; Expression *expression_; - const memgraph::query::plan::Filters all_filters_; + memgraph::query::plan::Filters all_filters_; static std::string SingleFilterName(const query::plan::FilterInfo &single_filter) { using Type = query::plan::FilterInfo::Type; if (single_filter.type == Type::Generic) { - std::set symbol_names; + std::set> symbol_names; for (const auto &symbol : single_filter.used_symbols) { symbol_names.insert(symbol.name()); } @@ -1144,7 +1142,7 @@ class Filter : public memgraph::query::plan::LogicalOperator { LOG_FATAL("Label filters not using LabelsTest are not supported for query inspection!"); } auto filter_expression = static_cast(single_filter.expression); - std::set label_names; + std::set> label_names; for (const auto &label : filter_expression->labels_) { label_names.insert(label.name); } @@ -1167,7 +1165,7 @@ class Filter : public memgraph::query::plan::LogicalOperator { } std::string ToString() const override { - std::set filter_names; + std::set> filter_names; for (const auto &filter : all_filters_) { filter_names.insert(Filter::SingleFilterName(filter)); } @@ -1214,7 +1212,7 @@ class Produce : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Produce() {} + Produce() = default; Produce(const std::shared_ptr &input, const std::vector &named_expressions); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1271,7 +1269,7 @@ class Delete : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Delete() {} + Delete() = default; Delete(const std::shared_ptr &input_, const std::vector &expressions, bool detach_); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1326,7 +1324,7 @@ class SetProperty : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - SetProperty() {} + SetProperty() = default; SetProperty(const std::shared_ptr &input, storage::PropertyId property, PropertyLookup *lhs, Expression *rhs); @@ -1385,7 +1383,7 @@ class SetProperties : public memgraph::query::plan::LogicalOperator { /// that the old properties are discarded and replaced with new ones. enum class Op { UPDATE, REPLACE }; - SetProperties() {} + SetProperties() = default; SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1433,7 +1431,7 @@ class SetLabels : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - SetLabels() {} + SetLabels() = default; SetLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels); @@ -1477,7 +1475,7 @@ class RemoveProperty : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - RemoveProperty() {} + RemoveProperty() = default; RemoveProperty(const std::shared_ptr &input, storage::PropertyId property, PropertyLookup *lhs); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1522,7 +1520,7 @@ class RemoveLabels : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - RemoveLabels() {} + RemoveLabels() = default; RemoveLabels(const std::shared_ptr &input, Symbol input_symbol, const std::vector &labels); @@ -1578,7 +1576,7 @@ class EdgeUniquenessFilter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EdgeUniquenessFilter() {} + EdgeUniquenessFilter() = default; EdgeUniquenessFilter(const std::shared_ptr &input, Symbol expand_symbol, const std::vector &previous_symbols); @@ -1636,7 +1634,7 @@ class EmptyResult : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EmptyResult() {} + EmptyResult() = default; EmptyResult(const std::shared_ptr &input); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1688,7 +1686,7 @@ class Accumulate : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Accumulate() {} + Accumulate() = default; Accumulate(const std::shared_ptr &input, const std::vector &symbols, bool advance_command = false); @@ -1811,7 +1809,7 @@ class Skip : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Skip() {} + Skip() = default; Skip(const std::shared_ptr &input, Expression *expression); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1857,7 +1855,7 @@ class EvaluatePatternFilter : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - EvaluatePatternFilter() {} + EvaluatePatternFilter() = default; EvaluatePatternFilter(const std::shared_ptr &input, Symbol output_symbol); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1911,7 +1909,7 @@ class Limit : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Limit() {} + Limit() = default; Limit(const std::shared_ptr &input, Expression *expression); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -1966,7 +1964,7 @@ class OrderBy : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OrderBy() {} + OrderBy() = default; OrderBy(const std::shared_ptr &input, const std::vector &order_by, const std::vector &output_symbols); @@ -2018,7 +2016,7 @@ class Merge : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Merge() {} + Merge() = default; Merge(const std::shared_ptr &input, const std::shared_ptr &merge_match, const std::shared_ptr &merge_create); @@ -2078,7 +2076,7 @@ class Optional : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Optional() {} + Optional() = default; Optional(const std::shared_ptr &input, const std::shared_ptr &optional, const std::vector &optional_symbols); @@ -2132,7 +2130,7 @@ class Unwind : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Unwind() {} + Unwind() = default; Unwind(const std::shared_ptr &input, Expression *input_expression_, Symbol output_symbol); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2167,7 +2165,7 @@ class Distinct : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Distinct() {} + Distinct() = default; Distinct(const std::shared_ptr &input, const std::vector &value_symbols); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2200,7 +2198,7 @@ class Union : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Union() {} + Union() = default; Union(const std::shared_ptr &left_op, const std::shared_ptr &right_op, const std::vector &union_symbols, const std::vector &left_symbols, @@ -2256,7 +2254,7 @@ class Cartesian : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Cartesian() {} + Cartesian() = default; /** Construct the operator with left input branch and right input branch. */ Cartesian(const std::shared_ptr &left_op, const std::vector &left_symbols, const std::shared_ptr &right_op, const std::vector &right_symbols) @@ -2291,7 +2289,7 @@ class OutputTable : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OutputTable() {} + OutputTable() = default; OutputTable(std::vector output_symbols, std::function>(Frame *, ExecutionContext *)> callback); OutputTable(std::vector output_symbols, std::vector> rows); @@ -2327,7 +2325,7 @@ class OutputTableStream : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - OutputTableStream() {} + OutputTableStream() = default; OutputTableStream(std::vector output_symbols, std::function>(Frame *, ExecutionContext *)> callback); @@ -2498,7 +2496,7 @@ class Apply : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - Apply() {} + Apply() = default; Apply(const std::shared_ptr input, const std::shared_ptr subquery, bool subquery_has_return); @@ -2545,7 +2543,7 @@ class IndexedJoin : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - IndexedJoin() {} + IndexedJoin() = default; IndexedJoin(std::shared_ptr main_branch, std::shared_ptr sub_branch); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; @@ -2588,7 +2586,7 @@ class HashJoin : public memgraph::query::plan::LogicalOperator { static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - HashJoin() {} + HashJoin() = default; /** Construct the operator with left input branch and right input branch. */ HashJoin(const std::shared_ptr &left_op, const std::vector &left_symbols, const std::shared_ptr &right_op, const std::vector &right_symbols, @@ -2631,5 +2629,4 @@ class HashJoin : public memgraph::query::plan::LogicalOperator { }; } // namespace plan -} // namespace query -} // namespace memgraph +} // namespace memgraph::query diff --git a/src/query/plan/planner.hpp b/src/query/plan/planner.hpp index 3f3d853bc..10318e6b9 100644 --- a/src/query/plan/planner.hpp +++ b/src/query/plan/planner.hpp @@ -17,6 +17,8 @@ #pragma once +#include + #include "query/plan/cost_estimator.hpp" #include "query/plan/operator.hpp" #include "query/plan/preprocess.hpp" @@ -42,11 +44,11 @@ class PostProcessor final { using ProcessedPlan = std::unique_ptr; - explicit PostProcessor(const Parameters ¶meters) : parameters_(parameters) {} + explicit PostProcessor(Parameters parameters) : parameters_(std::move(parameters)) {} template - PostProcessor(const Parameters ¶meters, std::vector index_hints, TDbAccessor *db) - : parameters_(parameters), index_hints_(IndexHints(index_hints, db)) {} + PostProcessor(Parameters parameters, std::vector index_hints, TDbAccessor *db) + : parameters_(std::move(parameters)), index_hints_(IndexHints(index_hints, db)) {} template std::unique_ptr Rewrite(std::unique_ptr plan, TPlanningContext *context) { diff --git a/src/query/plan/preprocess.cpp b/src/query/plan/preprocess.cpp index 8b1689796..e03c51841 100644 --- a/src/query/plan/preprocess.cpp +++ b/src/query/plan/preprocess.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "query/exceptions.hpp" @@ -199,7 +200,7 @@ auto SplitExpressionOnAnd(Expression *expression) { PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &symbol, PropertyIx property, Expression *value, Type type) - : symbol_(symbol), property_(property), type_(type), value_(value) { + : symbol_(symbol), property_(std::move(property)), type_(type), value_(value) { MG_ASSERT(type != Type::RANGE); UsedSymbolsCollector collector(symbol_table); value->Accept(collector); @@ -209,7 +210,11 @@ PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &sy PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &symbol, PropertyIx property, const std::optional &lower_bound, const std::optional &upper_bound) - : symbol_(symbol), property_(property), type_(Type::RANGE), lower_bound_(lower_bound), upper_bound_(upper_bound) { + : symbol_(symbol), + property_(std::move(property)), + type_(Type::RANGE), + lower_bound_(lower_bound), + upper_bound_(upper_bound) { UsedSymbolsCollector collector(symbol_table); if (lower_bound) { lower_bound->value()->Accept(collector); @@ -220,8 +225,8 @@ PropertyFilter::PropertyFilter(const SymbolTable &symbol_table, const Symbol &sy is_symbol_in_value_ = utils::Contains(collector.symbols_, symbol); } -PropertyFilter::PropertyFilter(const Symbol &symbol, PropertyIx property, Type type) - : symbol_(symbol), property_(property), type_(type) { +PropertyFilter::PropertyFilter(Symbol symbol, PropertyIx property, Type type) + : symbol_(std::move(symbol)), property_(std::move(property)), type_(type) { // As this constructor is used for property filters where // we don't have to evaluate the filter expression, we set // the is_symbol_in_value_ to false, although the filter diff --git a/src/query/plan/preprocess.hpp b/src/query/plan/preprocess.hpp index e9f7aeeb9..8e1955907 100644 --- a/src/query/plan/preprocess.hpp +++ b/src/query/plan/preprocess.hpp @@ -103,29 +103,29 @@ class UsedSymbolsCollector : public HierarchicalTreeVisitor { }; // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define PREPROCESS_DEFINE_ID_TYPE(name) \ - class name final { \ - private: \ - explicit name(uint64_t id) : id_(id) {} \ - \ - public: \ - /* Default constructor to allow serialization or preallocation. */ \ - name() = default; \ - \ - static name FromUint(uint64_t id) { return name(id); } \ - static name FromInt(int64_t id) { return name(utils::MemcpyCast(id)); } \ - uint64_t AsUint() const { return id_; } \ - int64_t AsInt() const { return utils::MemcpyCast(id_); } \ - \ - private: \ - uint64_t id_; \ - }; \ - static_assert(std::is_trivially_copyable::value, "query::plan::" #name " must be trivially copyable!"); \ - inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ - inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ - inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ - inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \ - inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \ +#define PREPROCESS_DEFINE_ID_TYPE(name) \ + class name final { \ + private: \ + explicit name(uint64_t id) : id_(id) {} \ + \ + public: \ + /* Default constructor to allow serialization or preallocation. */ \ + name() = default; \ + \ + static name FromUint(uint64_t id) { return name(id); } \ + static name FromInt(int64_t id) { return name(utils::MemcpyCast(id)); } \ + uint64_t AsUint() const { return id_; } \ + int64_t AsInt() const { return utils::MemcpyCast(id_); } \ + \ + private: \ + uint64_t id_; \ + }; \ + static_assert(std::is_trivially_copyable_v, "query::plan::" #name " must be trivially copyable!"); \ + inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ + inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ + inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ + inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \ + inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \ inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); } PREPROCESS_DEFINE_ID_TYPE(ExpansionGroupId); @@ -259,7 +259,7 @@ class PropertyFilter { /// Used for the "PROP IS NOT NULL" filter, and can be used for any /// property filter that doesn't need to use an expression to produce /// values that should be filtered further. - PropertyFilter(const Symbol &, PropertyIx, Type); + PropertyFilter(Symbol, PropertyIx, Type); /// Symbol whose property is looked up. Symbol symbol_; diff --git a/src/query/plan/profile.hpp b/src/query/plan/profile.hpp index 041a34ac9..e0d884449 100644 --- a/src/query/plan/profile.hpp +++ b/src/query/plan/profile.hpp @@ -18,9 +18,7 @@ #include "query/typed_value.hpp" -namespace memgraph::query { - -namespace plan { +namespace memgraph::query::plan { /** * Stores profiling statistics for a single logical operator. @@ -43,5 +41,4 @@ std::vector> ProfilingStatsToTable(const ProfilingStatsW nlohmann::json ProfilingStatsToJson(const ProfilingStatsWithTotalTime &stats); -} // namespace plan -} // namespace memgraph::query +} // namespace memgraph::query::plan diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 05361cf73..4054f8c12 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -84,7 +85,7 @@ template class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { public: IndexLookupRewriter(SymbolTable *symbol_table, AstStorage *ast_storage, TDbAccessor *db, IndexHints index_hints) - : symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), index_hints_(index_hints) {} + : symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), index_hints_(std::move(index_hints)) {} using HierarchicalLogicalOperatorVisitor::PostVisit; using HierarchicalLogicalOperatorVisitor::PreVisit; @@ -676,9 +677,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { if (!db_->LabelPropertyIndexExists(GetLabel(label), GetProperty(property))) { continue; } - candidate_indices.emplace_back(std::make_pair( + candidate_indices.emplace_back( IndexHint{.index_type_ = IndexHint::IndexType::LABEL_PROPERTY, .label_ = label, .property_ = property}, - filter)); + filter); candidate_index_lookup.insert({std::make_pair(label, property), filter}); } } diff --git a/src/query/plan/variable_start_planner.cpp b/src/query/plan/variable_start_planner.cpp index eb30226d8..1c230628a 100644 --- a/src/query/plan/variable_start_planner.cpp +++ b/src/query/plan/variable_start_planner.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "utils/flag_validation.hpp" #include "utils/logging.hpp" @@ -216,7 +217,7 @@ CartesianProduct VaryMultiMatchingStarts(const std::vector variants; variants.reserve(matchings.size()); for (const auto &matching : matchings) { - variants.emplace_back(VaryMatchingStart(matching, symbol_table)); + variants.emplace_back(matching, symbol_table); } return MakeCartesianProduct(std::move(variants)); } @@ -247,8 +248,7 @@ VaryQueryPartMatching::VaryQueryPartMatching(SingleQueryPart query_part, const S merge_matchings_(VaryMultiMatchingStarts(query_part_.merge_matching, symbol_table)), filter_matchings_(VaryFilterMatchingStarts(query_part_.matching, symbol_table)) {} -VaryQueryPartMatching::iterator::iterator(const SingleQueryPart &query_part, - VaryMatchingStart::iterator matchings_begin, +VaryQueryPartMatching::iterator::iterator(SingleQueryPart query_part, VaryMatchingStart::iterator matchings_begin, VaryMatchingStart::iterator matchings_end, CartesianProduct::iterator optional_begin, CartesianProduct::iterator optional_end, @@ -256,18 +256,18 @@ VaryQueryPartMatching::iterator::iterator(const SingleQueryPart &query_part, CartesianProduct::iterator merge_end, CartesianProduct::iterator filter_begin, CartesianProduct::iterator filter_end) - : current_query_part_(query_part), - matchings_it_(matchings_begin), - matchings_end_(matchings_end), + : current_query_part_(std::move(query_part)), + matchings_it_(std::move(matchings_begin)), + matchings_end_(std::move(matchings_end)), optional_it_(optional_begin), optional_begin_(optional_begin), - optional_end_(optional_end), + optional_end_(std::move(optional_end)), merge_it_(merge_begin), merge_begin_(merge_begin), - merge_end_(merge_end), + merge_end_(std::move(merge_end)), filter_it_(filter_begin), filter_begin_(filter_begin), - filter_end_(filter_end) { + filter_end_(std::move(filter_end)) { if (matchings_it_ != matchings_end_) { // Fill the query part with the first variation of matchings SetCurrentQueryPart(); diff --git a/src/query/plan/variable_start_planner.hpp b/src/query/plan/variable_start_planner.hpp index 96fd3b78d..6bdba71f3 100644 --- a/src/query/plan/variable_start_planner.hpp +++ b/src/query/plan/variable_start_planner.hpp @@ -49,16 +49,16 @@ class CartesianProduct { using TElement = typename decltype(begin_->begin())::value_type; public: - CartesianProduct(std::vector sets) + explicit CartesianProduct(std::vector sets) : original_sets_(std::move(sets)), begin_(original_sets_.begin()), end_(original_sets_.end()) {} class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef std::vector value_type; - typedef long difference_type; - typedef const std::vector &reference; - typedef const std::vector *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = std::vector; + using difference_type = long; + using reference = const std::vector &; + using pointer = const std::vector *; explicit iterator(CartesianProduct *self, bool is_done) : self_(self), is_done_(is_done) { if (is_done || self->begin_ == self->end_) { @@ -186,11 +186,11 @@ class VaryMatchingStart { class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef Matching value_type; - typedef long difference_type; - typedef const Matching &reference; - typedef const Matching *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = Matching; + using difference_type = long; + using reference = const Matching &; + using pointer = const Matching *; iterator(VaryMatchingStart *, bool); @@ -240,13 +240,13 @@ class VaryQueryPartMatching { class iterator { public: - typedef std::input_iterator_tag iterator_category; - typedef SingleQueryPart value_type; - typedef long difference_type; - typedef const SingleQueryPart &reference; - typedef const SingleQueryPart *pointer; + using iterator_category = std::input_iterator_tag; + using value_type = SingleQueryPart; + using difference_type = long; + using reference = const SingleQueryPart &; + using pointer = const SingleQueryPart *; - iterator(const SingleQueryPart &, VaryMatchingStart::iterator, VaryMatchingStart::iterator, + iterator(SingleQueryPart, VaryMatchingStart::iterator, VaryMatchingStart::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator, CartesianProduct::iterator); @@ -383,8 +383,8 @@ class VariableStartPlanner { /// @brief The result of plan generation is an iterable of roots to multiple /// generated operator trees. - using PlanResult = typename std::result_of::Plan)( - VariableStartPlanner, QueryParts &)>::type; + using PlanResult = std::result_of_t::Plan)( + VariableStartPlanner, QueryParts &)>; }; } // namespace memgraph::query::plan diff --git a/src/query/plan/vertex_count_cache.hpp b/src/query/plan/vertex_count_cache.hpp index ff19ee95a..4cfb2486b 100644 --- a/src/query/plan/vertex_count_cache.hpp +++ b/src/query/plan/vertex_count_cache.hpp @@ -27,7 +27,7 @@ namespace memgraph::query::plan { template class VertexCountCache { public: - VertexCountCache(TDbAccessor *db) : db_(db) {} + explicit VertexCountCache(TDbAccessor *db) : db_(db) {} auto NameToLabel(const std::string &name) { return db_->NameToLabel(name); } auto NameToProperty(const std::string &name) { return db_->NameToProperty(name); } @@ -88,7 +88,7 @@ class VertexCountCache { } private: - typedef std::pair LabelPropertyKey; + using LabelPropertyKey = std::pair; struct LabelPropertyHash { size_t operator()(const LabelPropertyKey &key) const { @@ -96,9 +96,8 @@ class VertexCountCache { } }; - typedef std::pair>, - std::optional>> - BoundsKey; + using BoundsKey = std::pair>, + std::optional>>; struct BoundsHash { size_t operator()(const BoundsKey &key) const { diff --git a/src/replication/include/replication/config.hpp b/src/replication/include/replication/config.hpp index ca0cd8f16..f98069955 100644 --- a/src/replication/include/replication/config.hpp +++ b/src/replication/include/replication/config.hpp @@ -34,8 +34,8 @@ struct ReplicationClientConfig { std::chrono::seconds replica_check_frequency{1}; struct SSL { - std::string key_file = ""; - std::string cert_file = ""; + std::string key_file; + std::string cert_file; friend bool operator==(const SSL &, const SSL &) = default; }; diff --git a/src/replication/include/replication/replication_server.hpp b/src/replication/include/replication/replication_server.hpp index 032312dcc..e9ca1b549 100644 --- a/src/replication/include/replication/replication_server.hpp +++ b/src/replication/include/replication/replication_server.hpp @@ -23,7 +23,7 @@ struct FrequentHeartbeatReq { static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); - FrequentHeartbeatReq() {} + FrequentHeartbeatReq() = default; }; struct FrequentHeartbeatRes { @@ -32,7 +32,7 @@ struct FrequentHeartbeatRes { static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); - FrequentHeartbeatRes() {} + FrequentHeartbeatRes() = default; explicit FrequentHeartbeatRes(bool success) : success(success) {} bool success; diff --git a/src/requests/requests.cpp b/src/requests/requests.cpp index d6c37bd61..8a871184a 100644 --- a/src/requests/requests.cpp +++ b/src/requests/requests.cpp @@ -35,7 +35,7 @@ bool RequestPostJson(const std::string &url, const nlohmann::json &data, int tim CURLcode res = CURLE_UNSUPPORTED_PROTOCOL; long response_code = 0; - struct curl_slist *headers = NULL; + struct curl_slist *headers = nullptr; std::string payload = data.dump(); std::string user_agent = fmt::format("memgraph/{}", gflags::VersionString()); diff --git a/src/rpc/client.hpp b/src/rpc/client.hpp index bd98afe89..f727391ac 100644 --- a/src/rpc/client.hpp +++ b/src/rpc/client.hpp @@ -55,7 +55,7 @@ class Client { StreamHandler(const StreamHandler &) = delete; StreamHandler &operator=(const StreamHandler &) = delete; - ~StreamHandler() {} + ~StreamHandler() = default; slk::Builder *GetBuilder() { return &req_builder_; } diff --git a/src/rpc/client_pool.hpp b/src/rpc/client_pool.hpp index 408b006a2..86123899d 100644 --- a/src/rpc/client_pool.hpp +++ b/src/rpc/client_pool.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -13,6 +13,7 @@ #include #include +#include #include "rpc/client.hpp" @@ -25,8 +26,8 @@ namespace memgraph::rpc { */ class ClientPool { public: - ClientPool(const io::network::Endpoint &endpoint, communication::ClientContext *context) - : endpoint_(endpoint), context_(context) {} + ClientPool(io::network::Endpoint endpoint, communication::ClientContext *context) + : endpoint_(std::move(endpoint)), context_(context) {} template typename TRequestResponse::Response Call(Args &&...args) { diff --git a/src/rpc/exceptions.hpp b/src/rpc/exceptions.hpp index b0eb6c329..346c53a9a 100644 --- a/src/rpc/exceptions.hpp +++ b/src/rpc/exceptions.hpp @@ -21,7 +21,7 @@ namespace memgraph::rpc { /// This exception always requires explicit handling. class RpcFailedException : public utils::BasicException { public: - RpcFailedException(std::string_view msg) : utils::BasicException(msg) {} + explicit RpcFailedException(std::string_view msg) : utils::BasicException(msg) {} SPECIALIZE_GET_EXCEPTION_NAME(RpcFailedException); }; diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index 933daaa7f..8bc77579b 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -11,6 +11,8 @@ #include "rpc/protocol.hpp" +#include + #include "rpc/messages.hpp" #include "rpc/server.hpp" #include "rpc/version.hpp" @@ -21,9 +23,9 @@ namespace memgraph::rpc { -Session::Session(Server *server, const io::network::Endpoint &endpoint, communication::InputStream *input_stream, +Session::Session(Server *server, io::network::Endpoint endpoint, communication::InputStream *input_stream, communication::OutputStream *output_stream) - : server_(server), endpoint_(endpoint), input_stream_(input_stream), output_stream_(output_stream) {} + : server_(server), endpoint_(std::move(endpoint)), input_stream_(input_stream), output_stream_(output_stream) {} void Session::Execute() { auto ret = slk::CheckStreamComplete(input_stream_->data(), input_stream_->size()); diff --git a/src/rpc/protocol.hpp b/src/rpc/protocol.hpp index f8b25664c..a1deea17c 100644 --- a/src/rpc/protocol.hpp +++ b/src/rpc/protocol.hpp @@ -48,7 +48,7 @@ class SessionException : public utils::BasicException { */ class Session { public: - Session(Server *server, const io::network::Endpoint &endpoint, communication::InputStream *input_stream, + Session(Server *server, io::network::Endpoint endpoint, communication::InputStream *input_stream, communication::OutputStream *output_stream); /** diff --git a/src/slk/serialization.hpp b/src/slk/serialization.hpp index 06628e229..9ca99527d 100644 --- a/src/slk/serialization.hpp +++ b/src/slk/serialization.hpp @@ -60,10 +60,10 @@ void Save(const std::vector &obj, Builder *builder); template void Load(std::vector *obj, Reader *reader); -template -void Save(const std::set &obj, Builder *builder); -template -void Load(std::set *obj, Reader *reader); +template +void Save(const std::set &obj, Builder *builder); +template +void Load(std::set *obj, Reader *reader); template void Save(const std::map &obj, Builder *builder); @@ -201,8 +201,8 @@ inline void Load(std::vector *obj, Reader *reader) { } } -template -inline void Save(const std::set &obj, Builder *builder) { +template +inline void Save(const std::set &obj, Builder *builder) { uint64_t size = obj.size(); Save(size, builder); for (const auto &item : obj) { @@ -210,8 +210,8 @@ inline void Save(const std::set &obj, Builder *builder) { } } -template -inline void Load(std::set *obj, Reader *reader) { +template +inline void Load(std::set *obj, Reader *reader) { uint64_t size = 0; Load(&size, reader); for (uint64_t i = 0; i < size; ++i) { @@ -273,7 +273,7 @@ inline void Load(std::unique_ptr *obj, Reader *reader) { // Prevent any loading which may potentially break class hierarchies. // Unfortunately, C++14 doesn't have (or I'm not aware of it) a trait for // checking whether some type has any derived or base classes. - static_assert(!std::is_polymorphic::value, + static_assert(!std::is_polymorphic_v, "Only non polymorphic types can be loaded generically from a " "pointer. Pass a custom load function as the 3rd argument."); bool exists = false; @@ -379,7 +379,7 @@ inline void Load(std::shared_ptr *obj, Reader *reader, std::vector::value, + static_assert(!std::is_polymorphic_v, "Only non polymorphic types can be loaded generically from a " "pointer. Pass a custom load function as the 4th argument."); bool exists = false; diff --git a/src/slk/streams.cpp b/src/slk/streams.cpp index 1346393e8..5125d635a 100644 --- a/src/slk/streams.cpp +++ b/src/slk/streams.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -12,12 +12,13 @@ #include "slk/streams.hpp" #include +#include #include "utils/logging.hpp" namespace memgraph::slk { -Builder::Builder(std::function write_func) : write_func_(write_func) {} +Builder::Builder(std::function write_func) : write_func_(std::move(write_func)) {} void Builder::Save(const uint8_t *data, uint64_t size) { size_t offset = 0; diff --git a/src/storage/v2/disk/label_property_index.cpp b/src/storage/v2/disk/label_property_index.cpp index 5e538559a..9a40f03d1 100644 --- a/src/storage/v2/disk/label_property_index.cpp +++ b/src/storage/v2/disk/label_property_index.cpp @@ -211,8 +211,7 @@ uint64_t DiskLabelPropertyIndex::ApproximateVertexCount( void DiskLabelPropertyIndex::LoadIndexInfo(const std::vector &keys) { for (const auto &label_property : keys) { std::vector label_property_split = utils::Split(label_property, ","); - index_.emplace( - std::make_pair(LabelId::FromString(label_property_split[0]), PropertyId::FromString(label_property_split[1]))); + index_.emplace(LabelId::FromString(label_property_split[0]), PropertyId::FromString(label_property_split[1])); } } diff --git a/src/storage/v2/disk/rocksdb_storage.hpp b/src/storage/v2/disk/rocksdb_storage.hpp index 0e55c2748..09200d38a 100644 --- a/src/storage/v2/disk/rocksdb_storage.hpp +++ b/src/storage/v2/disk/rocksdb_storage.hpp @@ -32,7 +32,7 @@ namespace memgraph::storage { /// Wraps RocksDB objects inside a struct. Vertex_chandle and edge_chandle are column family handles that may be /// nullptr. In that case client should take care about them. struct RocksDBStorage { - explicit RocksDBStorage() {} + explicit RocksDBStorage() = default; RocksDBStorage(const RocksDBStorage &) = delete; RocksDBStorage &operator=(const RocksDBStorage &) = delete; diff --git a/src/storage/v2/disk/unique_constraints.cpp b/src/storage/v2/disk/unique_constraints.cpp index e0ec3cf82..3c17530c2 100644 --- a/src/storage/v2/disk/unique_constraints.cpp +++ b/src/storage/v2/disk/unique_constraints.cpp @@ -344,7 +344,7 @@ void DiskUniqueConstraints::LoadUniqueConstraints(const std::vector for (int i = 1; i < key_parts.size(); i++) { properties.insert(PropertyId::FromString(key_parts[i])); } - constraints_.emplace(std::make_pair(label, properties)); + constraints_.emplace(label, properties); } } diff --git a/src/storage/v2/durability/serialization.hpp b/src/storage/v2/durability/serialization.hpp index 409293220..ca3a19e0f 100644 --- a/src/storage/v2/durability/serialization.hpp +++ b/src/storage/v2/durability/serialization.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -27,7 +27,7 @@ namespace memgraph::storage::durability { /// (e.g. file and network). class BaseEncoder { protected: - ~BaseEncoder() {} + ~BaseEncoder() = default; public: virtual void WriteMarker(Marker marker) = 0; @@ -84,7 +84,7 @@ class Encoder final : public BaseEncoder { /// (e.g. file and network). class BaseDecoder { protected: - ~BaseDecoder() {} + ~BaseDecoder() = default; public: virtual std::optional ReadMarker() = 0; diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index d4278dba9..52872222b 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -1212,7 +1212,7 @@ RecoveredSnapshot LoadSnapshotVersion15(const std::filesystem::path &path, utils spdlog::info("Recover connectivity."); recovery_info.vertex_batches.reserve(vertex_batches.size()); for (const auto batch : vertex_batches) { - recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count)); + recovery_info.vertex_batches.emplace_back(Gid::FromUint(0), batch.count); } std::atomic highest_edge_gid{0}; @@ -1505,7 +1505,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis spdlog::info("Recover connectivity."); recovery_info.vertex_batches.reserve(vertex_batches.size()); for (const auto batch : vertex_batches) { - recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count)); + recovery_info.vertex_batches.emplace_back(Gid::FromUint(0), batch.count); } std::atomic highest_edge_gid{0}; diff --git a/src/storage/v2/durability/wal.hpp b/src/storage/v2/durability/wal.hpp index 3f598cb99..8f6492ac7 100644 --- a/src/storage/v2/durability/wal.hpp +++ b/src/storage/v2/durability/wal.hpp @@ -108,7 +108,7 @@ struct WalDeltaData { struct { std::string label; - std::set properties; + std::set> properties; } operation_label_properties; struct { diff --git a/src/storage/v2/edge_accessor.hpp b/src/storage/v2/edge_accessor.hpp index 86912c2b6..365619149 100644 --- a/src/storage/v2/edge_accessor.hpp +++ b/src/storage/v2/edge_accessor.hpp @@ -110,7 +110,7 @@ class EdgeAccessor final { } // namespace memgraph::storage -static_assert(std::is_trivially_copyable::value, +static_assert(std::is_trivially_copyable_v, "storage::EdgeAccessor must be trivially copyable!"); namespace std { diff --git a/src/storage/v2/id_types.hpp b/src/storage/v2/id_types.hpp index 2f9577246..3f2c8aa40 100644 --- a/src/storage/v2/id_types.hpp +++ b/src/storage/v2/id_types.hpp @@ -42,7 +42,7 @@ namespace memgraph::storage { private: \ uint64_t id_; \ }; \ - static_assert(std::is_trivially_copyable::value, "storage::" #name " must be trivially copyable!"); \ + static_assert(std::is_trivially_copyable_v, "storage::" #name " must be trivially copyable!"); \ inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \ inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \ inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \ diff --git a/src/storage/v2/modified_edge.hpp b/src/storage/v2/modified_edge.hpp index e08ef4ce8..293c74f43 100644 --- a/src/storage/v2/modified_edge.hpp +++ b/src/storage/v2/modified_edge.hpp @@ -34,8 +34,7 @@ struct ModifiedEdgeInfo { EdgeRef edge_ref; }; -static_assert(std::is_trivially_copyable::value, - "storage::ModifiedEdgeInfo must be trivially copyable!"); +static_assert(std::is_trivially_copyable_v, "storage::ModifiedEdgeInfo must be trivially copyable!"); using ModifiedEdgesMap = std::unordered_map; diff --git a/src/storage/v2/property_store.cpp b/src/storage/v2/property_store.cpp index 65f08f4d9..530d5f5f6 100644 --- a/src/storage/v2/property_store.cpp +++ b/src/storage/v2/property_store.cpp @@ -179,7 +179,7 @@ class Writer { public: class MetadataHandle { public: - MetadataHandle() {} + MetadataHandle() = default; explicit MetadataHandle(uint8_t *value) : value_(value) {} @@ -195,7 +195,7 @@ class Writer { uint8_t *value_{nullptr}; }; - Writer() {} + Writer() = default; Writer(uint8_t *data, uint64_t size) : data_(data), size_(size) {} @@ -1311,7 +1311,7 @@ std::vector> PropertyStore: id_old_new_change.reserve(properties.size() + old_properties.size()); for (const auto &[prop_id, new_value] : properties) { if (!old_properties.contains(prop_id)) { - id_old_new_change.emplace_back(std::make_tuple(prop_id, PropertyValue(), new_value)); + id_old_new_change.emplace_back(prop_id, PropertyValue(), new_value); } } @@ -1319,7 +1319,7 @@ std::vector> PropertyStore: auto [it, inserted] = properties.emplace(old_key, old_value); if (!inserted) { auto &new_value = it->second; - id_old_new_change.emplace_back(std::make_tuple(it->first, old_value, new_value)); + id_old_new_change.emplace_back(it->first, old_value, new_value); } } diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index 817f47bcb..8cd8cb384 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -90,7 +90,7 @@ class ReplicationClient { auto GetStorageId() const -> std::string; void Start(); - void StartTransactionReplication(const uint64_t current_wal_seq_num); + void StartTransactionReplication(uint64_t current_wal_seq_num); // Replication clients can be removed at any point // so to avoid any complexity of checking if the client was removed whenever // we want to send part of transaction and to avoid adding some GC logic this diff --git a/src/storage/v2/replication/rpc.cpp b/src/storage/v2/replication/rpc.cpp index 1477aa108..b722dfebf 100644 --- a/src/storage/v2/replication/rpc.cpp +++ b/src/storage/v2/replication/rpc.cpp @@ -14,9 +14,7 @@ namespace memgraph { -namespace storage { - -namespace replication { +namespace storage::replication { void AppendDeltasReq::Save(const AppendDeltasReq &self, memgraph::slk::Builder *builder) { memgraph::slk::Save(self, builder); @@ -59,8 +57,7 @@ void TimestampRes::Save(const TimestampRes &self, memgraph::slk::Builder *builde } void TimestampRes::Load(TimestampRes *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); } -} // namespace replication -} // namespace storage +} // namespace storage::replication constexpr utils::TypeInfo storage::replication::AppendDeltasReq::kType{utils::TypeId::REP_APPEND_DELTAS_REQ, "AppendDeltasReq", nullptr}; diff --git a/src/storage/v2/replication/rpc.hpp b/src/storage/v2/replication/rpc.hpp index 1c0d425c8..9e2f0b35e 100644 --- a/src/storage/v2/replication/rpc.hpp +++ b/src/storage/v2/replication/rpc.hpp @@ -14,16 +14,13 @@ #include #include #include +#include #include "rpc/messages.hpp" #include "slk/serialization.hpp" #include "slk/streams.hpp" -namespace memgraph { - -namespace storage { - -namespace replication { +namespace memgraph::storage::replication { struct AppendDeltasReq { static const utils::TypeInfo kType; @@ -31,7 +28,7 @@ struct AppendDeltasReq { static void Load(AppendDeltasReq *self, memgraph::slk::Reader *reader); static void Save(const AppendDeltasReq &self, memgraph::slk::Builder *builder); - AppendDeltasReq() {} + AppendDeltasReq() = default; AppendDeltasReq(std::string name, uint64_t previous_commit_timestamp, uint64_t seq_num) : db_name(std::move(name)), previous_commit_timestamp(previous_commit_timestamp), seq_num(seq_num) {} @@ -46,7 +43,7 @@ struct AppendDeltasRes { static void Load(AppendDeltasRes *self, memgraph::slk::Reader *reader); static void Save(const AppendDeltasRes &self, memgraph::slk::Builder *builder); - AppendDeltasRes() {} + AppendDeltasRes() = default; AppendDeltasRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -63,7 +60,7 @@ struct HeartbeatReq { static void Load(HeartbeatReq *self, memgraph::slk::Reader *reader); static void Save(const HeartbeatReq &self, memgraph::slk::Builder *builder); - HeartbeatReq() {} + HeartbeatReq() = default; HeartbeatReq(std::string name, uint64_t main_commit_timestamp, std::string epoch_id) : db_name(std::move(name)), main_commit_timestamp(main_commit_timestamp), epoch_id(std::move(epoch_id)) {} @@ -78,12 +75,12 @@ struct HeartbeatRes { static void Load(HeartbeatRes *self, memgraph::slk::Reader *reader); static void Save(const HeartbeatRes &self, memgraph::slk::Builder *builder); - HeartbeatRes() {} + HeartbeatRes() = default; HeartbeatRes(std::string name, bool success, uint64_t current_commit_timestamp, std::string epoch_id) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp), - epoch_id(epoch_id) {} + epoch_id(std::move(epoch_id)) {} std::string db_name; bool success; @@ -99,7 +96,7 @@ struct SnapshotReq { static void Load(SnapshotReq *self, memgraph::slk::Reader *reader); static void Save(const SnapshotReq &self, memgraph::slk::Builder *builder); - SnapshotReq() {} + SnapshotReq() = default; explicit SnapshotReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -111,7 +108,7 @@ struct SnapshotRes { static void Load(SnapshotRes *self, memgraph::slk::Reader *reader); static void Save(const SnapshotRes &self, memgraph::slk::Builder *builder); - SnapshotRes() {} + SnapshotRes() = default; SnapshotRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -128,7 +125,7 @@ struct WalFilesReq { static void Load(WalFilesReq *self, memgraph::slk::Reader *reader); static void Save(const WalFilesReq &self, memgraph::slk::Builder *builder); - WalFilesReq() {} + WalFilesReq() = default; explicit WalFilesReq(std::string name, uint64_t file_number) : db_name(std::move(name)), file_number(file_number) {} std::string db_name; @@ -141,7 +138,7 @@ struct WalFilesRes { static void Load(WalFilesRes *self, memgraph::slk::Reader *reader); static void Save(const WalFilesRes &self, memgraph::slk::Builder *builder); - WalFilesRes() {} + WalFilesRes() = default; WalFilesRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -158,7 +155,7 @@ struct CurrentWalReq { static void Load(CurrentWalReq *self, memgraph::slk::Reader *reader); static void Save(const CurrentWalReq &self, memgraph::slk::Builder *builder); - CurrentWalReq() {} + CurrentWalReq() = default; explicit CurrentWalReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -170,7 +167,7 @@ struct CurrentWalRes { static void Load(CurrentWalRes *self, memgraph::slk::Reader *reader); static void Save(const CurrentWalRes &self, memgraph::slk::Builder *builder); - CurrentWalRes() {} + CurrentWalRes() = default; CurrentWalRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -187,7 +184,7 @@ struct TimestampReq { static void Load(TimestampReq *self, memgraph::slk::Reader *reader); static void Save(const TimestampReq &self, memgraph::slk::Builder *builder); - TimestampReq() {} + TimestampReq() = default; explicit TimestampReq(std::string name) : db_name(std::move(name)) {} std::string db_name; @@ -199,7 +196,7 @@ struct TimestampRes { static void Load(TimestampRes *self, memgraph::slk::Reader *reader); static void Save(const TimestampRes &self, memgraph::slk::Builder *builder); - TimestampRes() {} + TimestampRes() = default; TimestampRes(std::string name, bool success, uint64_t current_commit_timestamp) : db_name(std::move(name)), success(success), current_commit_timestamp(current_commit_timestamp) {} @@ -209,12 +206,9 @@ struct TimestampRes { }; using TimestampRpc = rpc::RequestResponse; -} // namespace replication -} // namespace storage -} // namespace memgraph +} // namespace memgraph::storage::replication // SLK serialization declarations -#include "slk/serialization.hpp" namespace memgraph::slk { void Save(const memgraph::storage::replication::TimestampRes &self, memgraph::slk::Builder *builder); diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index d91846ca4..8d8b06cd6 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -140,7 +140,7 @@ class Storage { Accessor(Accessor &&other) noexcept; - virtual ~Accessor() {} + virtual ~Accessor() = default; virtual VertexAccessor CreateVertex() = 0; diff --git a/src/storage/v2/vertex_accessor.hpp b/src/storage/v2/vertex_accessor.hpp index 8f67bc30b..0e5972d14 100644 --- a/src/storage/v2/vertex_accessor.hpp +++ b/src/storage/v2/vertex_accessor.hpp @@ -127,7 +127,7 @@ class VertexAccessor final { bool for_deleted_{false}; }; -static_assert(std::is_trivially_copyable::value, +static_assert(std::is_trivially_copyable_v, "storage::VertexAccessor must be trivially copyable!"); struct EdgesVertexAccessorResult { diff --git a/src/telemetry/telemetry.cpp b/src/telemetry/telemetry.cpp index 714788841..1635dda99 100644 --- a/src/telemetry/telemetry.cpp +++ b/src/telemetry/telemetry.cpp @@ -12,6 +12,7 @@ #include "telemetry/telemetry.hpp" #include +#include #include @@ -36,8 +37,8 @@ Telemetry::Telemetry(std::string url, std::filesystem::path storage_directory, s bool ssl, std::filesystem::path root_directory, std::chrono::duration refresh_interval, const uint64_t send_every_n) : url_(std::move(url)), - uuid_(uuid), - machine_id_(machine_id), + uuid_(std::move(uuid)), + machine_id_(std::move(machine_id)), ssl_(ssl), send_every_n_(send_every_n), storage_(std::move(storage_directory)) { diff --git a/src/utils/cast.hpp b/src/utils/cast.hpp index bdb8facc6..f015caf94 100644 --- a/src/utils/cast.hpp +++ b/src/utils/cast.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -18,8 +18,8 @@ namespace memgraph::utils { template -constexpr typename std::underlying_type::type UnderlyingCast(T e) { - return static_cast::type>(e); +constexpr std::underlying_type_t UnderlyingCast(T e) { + return static_cast>(e); } /** @@ -36,8 +36,8 @@ template TDest MemcpyCast(TSrc src) { TDest dest; static_assert(sizeof(dest) == sizeof(src), "MemcpyCast expects source and destination to be of same size"); - static_assert(std::is_arithmetic::value, "MemcpyCast expects source is an arithmetic type"); - static_assert(std::is_arithmetic::value, "MemcypCast expects destination is an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcpyCast expects source is an arithmetic type"); + static_assert(std::is_arithmetic_v, "MemcypCast expects destination is an arithmetic type"); std::memcpy(&dest, &src, sizeof(src)); return dest; } diff --git a/src/utils/event_histogram.hpp b/src/utils/event_histogram.hpp index 3768f0ec1..02d612b60 100644 --- a/src/utils/event_histogram.hpp +++ b/src/utils/event_histogram.hpp @@ -12,6 +12,7 @@ #pragma once #include +#include #include "utils/logging.hpp" @@ -73,7 +74,9 @@ class Histogram { percentiles_ = {0, 25, 50, 75, 90, 100}; } - explicit Histogram(std::vector percentiles) : percentiles_(percentiles) { samples_.resize(kSampleLimit, 0); } + explicit Histogram(std::vector percentiles) : percentiles_(std::move(percentiles)) { + samples_.resize(kSampleLimit, 0); + } uint64_t Count() const { return count_.load(std::memory_order_relaxed); } @@ -104,7 +107,7 @@ class Histogram { percentile_yield.reserve(percentiles_.size()); for (const auto percentile : percentiles_) { - percentile_yield.emplace_back(std::make_pair(percentile, Percentile(percentile))); + percentile_yield.emplace_back(percentile, Percentile(percentile)); } return percentile_yield; diff --git a/src/utils/exceptions.hpp b/src/utils/exceptions.hpp index d9a927beb..fa49f770e 100644 --- a/src/utils/exceptions.hpp +++ b/src/utils/exceptions.hpp @@ -61,7 +61,7 @@ class BasicException : public std::exception { /** * @brief Virtual destructor to allow for subclassing. */ - virtual ~BasicException() {} + ~BasicException() override = default; /** * @brief Returns a pointer to the (constant) error description. @@ -116,7 +116,7 @@ class StacktraceException : public std::exception { /** * @brief Virtual destructor to allow for subclassing. */ - virtual ~StacktraceException() {} + ~StacktraceException() override = default; /** * @brief Returns a pointer to the (constant) error description. diff --git a/src/utils/logging.hpp b/src/utils/logging.hpp index 9b9e7705b..02389beab 100644 --- a/src/utils/logging.hpp +++ b/src/utils/logging.hpp @@ -47,17 +47,21 @@ inline void AssertFailed(const char *file_name, int line_num, const char *expr, #define GET_MESSAGE(...) \ BOOST_PP_IF(BOOST_PP_EQUAL(BOOST_PP_VARIADIC_SIZE(__VA_ARGS__), 0), "", fmt::format(__VA_ARGS__)) -#define MG_ASSERT(expr, ...) \ - if (expr) [[likely]] { \ - (void)0; \ - } else { \ - ::memgraph::logging::AssertFailed(__FILE__, __LINE__, #expr, GET_MESSAGE(__VA_ARGS__)); \ - } +#define MG_ASSERT(expr, ...) \ + do { \ + if (expr) [[likely]] { \ + (void)0; \ + } else { \ + ::memgraph::logging::AssertFailed(__FILE__, __LINE__, #expr, GET_MESSAGE(__VA_ARGS__)); \ + } \ + } while (false) #ifndef NDEBUG #define DMG_ASSERT(expr, ...) MG_ASSERT(expr, __VA_ARGS__) #else -#define DMG_ASSERT(...) +#define DMG_ASSERT(...) \ + do { \ + } while (false) #endif template @@ -75,7 +79,9 @@ void Fatal(const char *msg, const Args &...msg_args) { #ifndef NDEBUG #define DLOG_FATAL(...) LOG_FATAL(__VA_ARGS__) #else -#define DLOG_FATAL(...) +#define DLOG_FATAL(...) \ + do { \ + } while (false) #endif inline void RedirectToStderr() { spdlog::set_default_logger(spdlog::stderr_color_mt("stderr")); } diff --git a/src/utils/lru_cache.hpp b/src/utils/lru_cache.hpp index 1ab636670..f94365331 100644 --- a/src/utils/lru_cache.hpp +++ b/src/utils/lru_cache.hpp @@ -24,7 +24,7 @@ namespace memgraph::utils { template class LRUCache { public: - LRUCache(int cache_size_) : cache_size(cache_size_){}; + explicit LRUCache(int cache_size_) : cache_size(cache_size_){}; void put(const TKey &key, const TVal &val) { auto it = item_map.find(key); diff --git a/src/utils/memory.hpp b/src/utils/memory.hpp index 3bec47ec0..225a3b6a1 100644 --- a/src/utils/memory.hpp +++ b/src/utils/memory.hpp @@ -45,7 +45,7 @@ class BadAlloc final : public std::bad_alloc { std::string msg_; public: - explicit BadAlloc(const std::string &msg) : msg_(msg) {} + explicit BadAlloc(std::string msg) : msg_(std::move(msg)) {} const char *what() const noexcept override { return msg_.c_str(); } }; @@ -53,7 +53,7 @@ class BadAlloc final : public std::bad_alloc { /// Abstract class for writing custom memory management, i.e. allocators. class MemoryResource { public: - virtual ~MemoryResource() {} + virtual ~MemoryResource() = default; /// Allocate storage with a size of at least `bytes` bytes. /// diff --git a/src/utils/scheduler.hpp b/src/utils/scheduler.hpp index b97320c21..d96178598 100644 --- a/src/utils/scheduler.hpp +++ b/src/utils/scheduler.hpp @@ -28,7 +28,7 @@ namespace memgraph::utils { */ class Scheduler { public: - Scheduler() {} + Scheduler() = default; /** * @param pause - Duration between two function executions. If function is * still running when it should be ran again, it will run right after it diff --git a/src/utils/skip_list.hpp b/src/utils/skip_list.hpp index e2079b5be..de4892375 100644 --- a/src/utils/skip_list.hpp +++ b/src/utils/skip_list.hpp @@ -1091,8 +1091,8 @@ class SkipList final : detail::SkipListNode_base { if (lower) { layer_found = find_node(lower->value(), preds, succs); } else { - for (int i = 0; i < kSkipListMaxHeight; ++i) { - preds[i] = head_; + for (auto &pred : preds) { + pred = head_; } layer_found = kSkipListMaxHeight - 1; } diff --git a/src/utils/small_vector.hpp b/src/utils/small_vector.hpp index 8654c1849..d6d7a8dd2 100644 --- a/src/utils/small_vector.hpp +++ b/src/utils/small_vector.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -105,12 +105,12 @@ class SmallVectorTemplateCommon : public SmallVectorBase { // don't want it to be automatically run, so we need to represent the space as // something else. Use an array of char of sufficient alignment. ////////////typedef utils::AlignedCharArrayUnion U; - typedef typename std::aligned_union<1, T>::type U; + using U = typename std::aligned_union<1, T>::type; U first_el_; // Space after 'first_el' is clobbered, do not add any instance vars after it. protected: - SmallVectorTemplateCommon(size_t size) : SmallVectorBase(&first_el_, size) {} + explicit SmallVectorTemplateCommon(size_t size) : SmallVectorBase(&first_el_, size) {} void GrowPod(size_t min_size_in_bytes, size_t t_size) { SmallVectorBase::GrowPod(&first_el_, min_size_in_bytes, t_size); @@ -126,19 +126,19 @@ class SmallVectorTemplateCommon : public SmallVectorBase { void SetEnd(T *P) { this->end_x_ = P; } public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T *iterator; - typedef const T *const_iterator; + using size_type = size_t; + using difference_type = ptrdiff_t; + using value_type = T; + using iterator = T *; + using const_iterator = const T *; - typedef std::reverse_iterator const_reverse_iterator; - typedef std::reverse_iterator reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using reverse_iterator = std::reverse_iterator; - typedef T &reference; - typedef const T &const_reference; - typedef T *pointer; - typedef const T *const_pointer; + using reference = T &; + using const_reference = const T &; + using pointer = T *; + using const_pointer = const T *; // forward iterator creation methods. inline iterator begin() { return (iterator)this->begin_x_; } @@ -201,7 +201,7 @@ class SmallVectorTemplateCommon : public SmallVectorBase { template class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: - SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} + explicit SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} static void DestroyRange(T *s, T *e) { while (s != e) { @@ -277,7 +277,7 @@ void SmallVectorTemplateBase::Grow(size_t min_size) { template class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: - SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} + explicit SmallVectorTemplateBase(size_t size) : SmallVectorTemplateCommon(size) {} // No need to do a destroy loop for POD's. static void DestroyRange(T *, T *) {} @@ -331,14 +331,14 @@ inline constexpr bool is_pod = std::is_standard_layout_v &&std::is_trivial_v< /// reduce code duplication based on the SmallVector 'n' template parameter. template class SmallVectorImpl : public SmallVectorTemplateBase> { - typedef SmallVectorTemplateBase> SuperClass; + using SuperClass = SmallVectorTemplateBase>; SmallVectorImpl(const SmallVectorImpl &) = delete; public: - typedef typename SuperClass::iterator iterator; - typedef typename SuperClass::const_iterator const_iterator; - typedef typename SuperClass::size_type size_type; + using iterator = typename SuperClass::iterator; + using const_iterator = typename SuperClass::const_iterator; + using size_type = typename SuperClass::size_type; protected: // Default ctor - Initialize to empty. @@ -861,7 +861,7 @@ class SmallVector : public SmallVectorImpl { return *this; } - SmallVector(SmallVectorImpl &&rhs) : SmallVectorImpl(N) { + explicit SmallVector(SmallVectorImpl &&rhs) : SmallVectorImpl(N) { if (!rhs.empty()) SmallVectorImpl::operator=(::std::move(rhs)); } diff --git a/src/utils/stacktrace.hpp b/src/utils/stacktrace.hpp index a2aa5a6c3..f06152e7d 100644 --- a/src/utils/stacktrace.hpp +++ b/src/utils/stacktrace.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -15,6 +15,7 @@ #include #include #include +#include #include "utils/on_scope_exit.hpp" @@ -25,10 +26,10 @@ class Stacktrace { class Line { public: // cppcheck-suppress noExplicitConstructor - Line(const std::string &original) : original(original) {} + explicit Line(std::string original) : original(std::move(original)) {} - Line(const std::string &original, const std::string &function, const std::string &location) - : original(original), function(function), location(location) {} + Line(std::string original, std::string function, std::string location) + : original(std::move(original)), function(std::move(function)), location(std::move(location)) {} std::string original, function, location; }; @@ -84,7 +85,7 @@ class Stacktrace { auto begin = line.find('('); auto end = line.find('+'); - if (begin == std::string::npos || end == std::string::npos) return {original}; + if (begin == std::string::npos || end == std::string::npos) return Line{original}; line[end] = '\0'; diff --git a/src/utils/variant_helpers.hpp b/src/utils/variant_helpers.hpp index 6a58deafb..b7751fe3e 100644 --- a/src/utils/variant_helpers.hpp +++ b/src/utils/variant_helpers.hpp @@ -26,7 +26,7 @@ Overloaded(Ts...) -> Overloaded; template struct ChainedOverloaded : Ts... { template - ChainedOverloaded(Us &&...ts) : Ts(std::forward(ts))... {} + explicit ChainedOverloaded(Us &&...ts) : Ts(std::forward(ts))... {} template auto operator()(Args &&...args) { diff --git a/tests/benchmark/query/profile.cpp b/tests/benchmark/query/profile.cpp index cfa346fbf..1c28fd21c 100644 --- a/tests/benchmark/query/profile.cpp +++ b/tests/benchmark/query/profile.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -233,9 +233,9 @@ struct ScopedProfile { stats = nullptr; // Was this logical operator already hit on one of the previous pulls? - for (size_t i = 0; i < root->children.size(); ++i) { - if (root->children[i].key == key) { - stats = &root->children[i]; + for (auto &child : root->children) { + if (child.key == key) { + stats = &child; break; } } @@ -345,9 +345,9 @@ struct ScopedProfile { stats = nullptr; // Was this logical operator already hit on one of the previous pulls? - for (size_t i = 0; i < root->children.size(); ++i) { - if (root->children[i].key == key) { - stats = &root->children[i]; + for (auto &child : root->children) { + if (child.key == key) { + stats = &child; break; } } diff --git a/tests/benchmark/rpc.cpp b/tests/benchmark/rpc.cpp index 3dace0531..0bb1da4d3 100644 --- a/tests/benchmark/rpc.cpp +++ b/tests/benchmark/rpc.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -24,8 +25,8 @@ struct EchoMessage { static const memgraph::utils::TypeInfo kType; - EchoMessage() {} // Needed for serialization. - explicit EchoMessage(const std::string &data) : data(data) {} + EchoMessage() = default; // Needed for serialization. + explicit EchoMessage(std::string data) : data(std::move(data)) {} static void Load(EchoMessage *obj, memgraph::slk::Reader *reader); static void Save(const EchoMessage &obj, memgraph::slk::Builder *builder); diff --git a/tests/e2e/isolation_levels/isolation_levels.cpp b/tests/e2e/isolation_levels/isolation_levels.cpp index 15151b106..2ead05750 100644 --- a/tests/e2e/isolation_levels/isolation_levels.cpp +++ b/tests/e2e/isolation_levels/isolation_levels.cpp @@ -91,7 +91,7 @@ void SwitchToSameDB(std::unique_ptr &main, std::unique_ptrFetchAll(); MG_ASSERT(dbs, "Failed to show databases"); for (const auto &elem : *dbs) { - MG_ASSERT(elem.size(), "Show databases wrong output"); + MG_ASSERT(!elem.empty(), "Show databases wrong output"); const auto &active = elem[1].ValueString(); if (active == "*") { const auto &name = elem[0].ValueString(); diff --git a/tests/e2e/memory/memory_limit_global_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_alloc_proc.cpp index e1f530123..34aeae509 100644 --- a/tests/e2e/memory/memory_limit_global_alloc_proc.cpp +++ b/tests/e2e/memory/memory_limit_global_alloc_proc.cpp @@ -57,6 +57,6 @@ int main(int argc, char **argv) { MG_ASSERT(client->Execute("CALL libglobal_memory_limit_proc.success() YIELD *")); auto result2 = client->FetchAll(); - MG_ASSERT(result2 != std::nullopt && result2->size() > 0); + MG_ASSERT(result2 != std::nullopt && !result2->empty()); return 0; } diff --git a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp index 2132fbb16..e44c91ea7 100644 --- a/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp +++ b/tests/e2e/memory/memory_limit_global_multi_thread_proc_create.cpp @@ -54,7 +54,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp index 0c2eb1ee6..e9892ac82 100644 --- a/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp +++ b/tests/e2e/memory/memory_limit_global_thread_alloc_proc.cpp @@ -55,7 +55,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/procedure_memory_limit.cpp b/tests/e2e/memory/procedure_memory_limit.cpp index 89a91a9ce..07fbd17b2 100644 --- a/tests/e2e/memory/procedure_memory_limit.cpp +++ b/tests/e2e/memory/procedure_memory_limit.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/query_memory_limit_proc.cpp b/tests/e2e/memory/query_memory_limit_proc.cpp index 39baf4d8e..38509a766 100644 --- a/tests/e2e/memory/query_memory_limit_proc.cpp +++ b/tests/e2e/memory/query_memory_limit_proc.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp b/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp index 5a5ec94f0..5acac5404 100644 --- a/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp +++ b/tests/e2e/memory/query_memory_limit_proc_multi_thread.cpp @@ -53,7 +53,7 @@ int main(int argc, char **argv) { auto result_rows = client->FetchAll(); if (result_rows) { auto row = *result_rows->begin(); - error = row[0].ValueBool() == false; + error = !row[0].ValueBool(); } } catch (const std::exception &e) { diff --git a/tests/e2e/replication/common.hpp b/tests/e2e/replication/common.hpp index f3cff293e..f5113ac37 100644 --- a/tests/e2e/replication/common.hpp +++ b/tests/e2e/replication/common.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -39,7 +39,7 @@ auto ParseDatabaseEndpoints(const std::string &database_endpoints_str) { for (const auto &db_endpoint_str : db_endpoints_strs) { const auto maybe_host_port = memgraph::io::network::Endpoint::ParseSocketOrIpAddress(db_endpoint_str, 7687); MG_ASSERT(maybe_host_port); - database_endpoints.emplace_back(memgraph::io::network::Endpoint(maybe_host_port->first, maybe_host_port->second)); + database_endpoints.emplace_back(maybe_host_port->first, maybe_host_port->second); } return database_endpoints; } diff --git a/tests/e2e/replication/constraints.cpp b/tests/e2e/replication/constraints.cpp index f56e97946..01c1217f2 100644 --- a/tests/e2e/replication/constraints.cpp +++ b/tests/e2e/replication/constraints.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -132,7 +132,7 @@ int main(int argc, char **argv) { auto client = mg::e2e::replication::Connect(database_endpoint); client->Execute("SHOW CONSTRAINT INFO;"); if (const auto data = client->FetchAll()) { - if ((*data).size() != 0) { + if (!(*data).empty()) { LOG_FATAL("{} still have some constraints.", database_endpoint); } } else { diff --git a/tests/e2e/replication/read_write_benchmark.cpp b/tests/e2e/replication/read_write_benchmark.cpp index df1c4fb90..243aab2a8 100644 --- a/tests/e2e/replication/read_write_benchmark.cpp +++ b/tests/e2e/replication/read_write_benchmark.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -138,7 +138,7 @@ int main(int argc, char **argv) { auto client = mg::e2e::replication::Connect(database_endpoint); client->Execute("SHOW INDEX INFO;"); if (const auto data = client->FetchAll()) { - if ((*data).size() != 0) { + if (!(*data).empty()) { LOG_FATAL("{} still have some indexes.", database_endpoint); } } else { diff --git a/tests/e2e/triggers/common.hpp b/tests/e2e/triggers/common.hpp index fbffd19f3..74593c1b9 100644 --- a/tests/e2e/triggers/common.hpp +++ b/tests/e2e/triggers/common.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -22,7 +22,7 @@ inline constexpr std::string_view kVertexLabel{"VERTEX"}; inline constexpr std::string_view kEdgeLabel{"EDGE"}; std::unique_ptr Connect(); -std::unique_ptr ConnectWithUser(const std::string_view username); +std::unique_ptr ConnectWithUser(std::string_view username); void CreateVertex(mg::Client &client, int vertex_id); void CreateEdge(mg::Client &client, int from_vertex, int to_vertex, int edge_id); diff --git a/tests/manual/antlr_tree_pretty_print.cpp b/tests/manual/antlr_tree_pretty_print.cpp index 0ba057f34..8957fc7e7 100644 --- a/tests/manual/antlr_tree_pretty_print.cpp +++ b/tests/manual/antlr_tree_pretty_print.cpp @@ -1,4 +1,4 @@ -// Copyright 2021 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -40,7 +40,7 @@ int main(int, const char **) { CommonTokenStream tokens(&lexer); tokens.fill(); - for (auto token : tokens.getTokens()) { + for (auto *token : tokens.getTokens()) { std::cout << token->toString() << std::endl; } diff --git a/tests/stress/long_running.cpp b/tests/stress/long_running.cpp index edf4b8452..37ed562ce 100644 --- a/tests/stress/long_running.cpp +++ b/tests/stress/long_running.cpp @@ -86,7 +86,7 @@ class GraphSession { std::set edges_; std::string indexed_label_; - std::set labels_; + std::set> labels_; std::map> labels_vertices_; @@ -109,7 +109,7 @@ class GraphSession { return *it; } - std::string RandomElement(std::set &data) { + std::string RandomElement(std::set> &data) { uint64_t pos = std::floor(GetRandom() * data.size()); auto it = data.begin(); std::advance(it, pos); diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index b7a90f22d..956cba781 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -6,7 +6,8 @@ find_package(Threads REQUIRED) add_custom_target(memgraph__unit) -set(memgraph_unit_main main.cpp) +add_library(memgraph_unit_main OBJECT main.cpp) +target_link_libraries(memgraph_unit_main mg-memory mg-utils gtest gmock Threads::Threads dl) function(add_unit_test test_cpp) _add_unit_test(${test_cpp} FALSE ${ARGN}) @@ -25,19 +26,19 @@ function(_add_unit_test test_cpp custom_main) ${test_cpp} ${ARGN}) - if(NOT ${custom_main}) - set(source_files - ${source_files} - ${memgraph_unit_main}) - endif() - add_executable(${target_name} ${source_files}) + if(NOT ${custom_main}) + target_link_libraries(${target_name} memgraph_unit_main) + else() + target_link_libraries(${target_name} gtest gmock Threads::Threads dl) + endif() + # OUTPUT_NAME sets the real name of a target when it is built and can be # used to help create two targets of the same name even though CMake # requires unique logical target names set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) - target_link_libraries(${target_name} mg-memory mg-utils gtest gmock Threads::Threads dl) + # register test if(TEST_COVERAGE) diff --git a/tests/unit/auth.cpp b/tests/unit/auth.cpp index 86a794480..6dbe20914 100644 --- a/tests/unit/auth.cpp +++ b/tests/unit/auth.cpp @@ -32,7 +32,7 @@ DECLARE_string(password_encryption_algorithm); class AuthWithStorage : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -40,7 +40,7 @@ class AuthWithStorage : public ::testing::Test { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; @@ -58,7 +58,7 @@ TEST_F(AuthWithStorage, RemoveRole) { ASSERT_TRUE(auth.RemoveRole("admin")); class AuthWithStorage : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -66,7 +66,7 @@ TEST_F(AuthWithStorage, RemoveRole) { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; @@ -589,9 +589,9 @@ TEST(AuthWithoutStorage, FineGrainedAccessPermissions) { } TEST_F(AuthWithStorage, FineGrainedAccessCheckerMerge) { - auto any_label = "AnyString"; - auto check_label = "Label"; - auto asterisk = "*"; + const auto *any_label = "AnyString"; + const auto *check_label = "Label"; + const auto *asterisk = "*"; { FineGrainedAccessPermissions fga_permissions1, fga_permissions2; @@ -929,7 +929,7 @@ TEST(AuthWithoutStorage, Crypto) { class AuthWithVariousEncryptionAlgorithms : public ::testing::Test { protected: - virtual void SetUp() { FLAGS_password_encryption_algorithm = "bcrypt"; } + void SetUp() override { FLAGS_password_encryption_algorithm = "bcrypt"; } }; TEST_F(AuthWithVariousEncryptionAlgorithms, VerifyPasswordDefault) { @@ -964,7 +964,7 @@ TEST_F(AuthWithVariousEncryptionAlgorithms, VerifyPasswordEmptyEncryptionThrow) class AuthWithStorageWithVariousEncryptionAlgorithms : public ::testing::Test { protected: - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); FLAGS_auth_password_permit_null = true; FLAGS_auth_password_strength_regex = ".+"; @@ -973,7 +973,7 @@ class AuthWithStorageWithVariousEncryptionAlgorithms : public ::testing::Test { memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / "MG_tests_unit_auth"}; diff --git a/tests/unit/auth_handler.cpp b/tests/unit/auth_handler.cpp index e3352220f..6537575fd 100644 --- a/tests/unit/auth_handler.cpp +++ b/tests/unit/auth_handler.cpp @@ -36,12 +36,12 @@ class AuthQueryHandlerFixture : public testing::Test { #ifdef MG_ENTERPRISE memgraph::auth::FineGrainedAccessHandler handler{}; #endif - virtual void SetUp() { + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); memgraph::license::global_license_checker.EnableTesting(); } - virtual void TearDown() { + void TearDown() override { std::filesystem::remove_all(test_folder_); perms = memgraph::auth::Permissions{}; #ifdef MG_ENTERPRISE diff --git a/tests/unit/bfs_common.hpp b/tests/unit/bfs_common.hpp index 3b16e4141..9bed67971 100644 --- a/tests/unit/bfs_common.hpp +++ b/tests/unit/bfs_common.hpp @@ -288,7 +288,7 @@ class Database { virtual std::pair, std::vector> BuildGraph(memgraph::query::DbAccessor *dba, const std::vector &vertex_locations, const std::vector> &edges) = 0; - virtual ~Database() {} + virtual ~Database() = default; void BfsTest(Database *db, int lower_bound, int upper_bound, memgraph::query::EdgeAtom::Direction direction, std::vector edge_types, bool known_sink, FilterLambdaType filter_lambda_type) { diff --git a/tests/unit/bolt_chunked_encoder_buffer.cpp b/tests/unit/bolt_chunked_encoder_buffer.cpp index 8dd23a0ff..47238b403 100644 --- a/tests/unit/bolt_chunked_encoder_buffer.cpp +++ b/tests/unit/bolt_chunked_encoder_buffer.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -87,7 +87,7 @@ TEST_F(BoltChunkedEncoderBuffer, TwoSmallChunks) { // the output array should look like this: // [0, 100, first 100 bytes of test data] + // [0, 100, second 100 bytes of test data] - auto data = output_stream.output.data(); + auto *data = output_stream.output.data(); VerifyChunkOfTestData(data, size1); VerifyChunkOfTestData(data + kChunkHeaderSize + size1, size2, size1); } @@ -105,7 +105,7 @@ TEST_F(BoltChunkedEncoderBuffer, OneAndAHalfOfMaxChunk) { // the output array should look like this: // [0xFF, 0xFF, first 65535 bytes of test data, // 0x86, 0xA1, 34465 bytes of test data after the first 65535 bytes] - auto output = output_stream.output.data(); + auto *output = output_stream.output.data(); VerifyChunkOfTestData(output, kChunkMaxDataSize); VerifyChunkOfTestData(output + kChunkWholeSize, kTestDataSize - kChunkMaxDataSize, kChunkMaxDataSize); } diff --git a/tests/unit/cpp_api.cpp b/tests/unit/cpp_api.cpp index ecad43487..929a70431 100644 --- a/tests/unit/cpp_api.cpp +++ b/tests/unit/cpp_api.cpp @@ -27,7 +27,7 @@ template struct CppApiTestFixture : public ::testing::Test { protected: - virtual void SetUp() override { mgp::mrd.Register(&memory); } + void SetUp() override { mgp::mrd.Register(&memory); } void TearDown() override { if (std::is_same::value) { diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index 1d7f236a2..54453de09 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -58,7 +58,7 @@ class Base { ParsingContext context_; Parameters parameters_; - virtual ~Base() {} + virtual ~Base() = default; virtual Query *ParseQuery(const std::string &query_string) = 0; @@ -199,8 +199,8 @@ class CachedAstGenerator : public Base { class MockModule : public procedure::Module { public: - MockModule(){}; - ~MockModule() override{}; + MockModule() = default; + ~MockModule() override = default; MockModule(const MockModule &) = delete; MockModule(MockModule &&) = delete; MockModule &operator=(const MockModule &) = delete; diff --git a/tests/unit/integrations_kafka_consumer.cpp b/tests/unit/integrations_kafka_consumer.cpp index 63a419552..3d5feb80b 100644 --- a/tests/unit/integrations_kafka_consumer.cpp +++ b/tests/unit/integrations_kafka_consumer.cpp @@ -48,7 +48,7 @@ inline constexpr int64_t kDefaultBatchSize{1000}; } // namespace struct ConsumerTest : public ::testing::Test { - ConsumerTest() {} + ConsumerTest() = default; ConsumerInfo CreateDefaultConsumerInfo() const { const auto test_name = std::string{::testing::UnitTest::GetInstance()->current_test_info()->name()}; @@ -132,7 +132,7 @@ TEST_F(ConsumerTest, BatchInterval) { info.batch_interval = kBatchInterval; auto expected_messages_received = true; auto consumer_function = [&](const std::vector &messages) mutable { - received_timestamps.push_back({messages.size(), std::chrono::steady_clock::now()}); + received_timestamps.emplace_back(messages.size(), std::chrono::steady_clock::now()); for (const auto &message : messages) { expected_messages_received &= (kMessage == std::string_view(message.Payload().data(), message.Payload().size())); } @@ -227,7 +227,7 @@ TEST_F(ConsumerTest, BatchSize) { static constexpr std::string_view kMessage = "BatchSizeTestMessage"; auto expected_messages_received = true; auto consumer_function = [&](const std::vector &messages) mutable { - received_timestamps.push_back({messages.size(), std::chrono::steady_clock::now()}); + received_timestamps.emplace_back(messages.size(), std::chrono::steady_clock::now()); for (const auto &message : messages) { expected_messages_received &= (kMessage == std::string_view(message.Payload().data(), message.Payload().size())); } diff --git a/tests/unit/interpreter.cpp b/tests/unit/interpreter.cpp index 5eb7cd539..57bb79db8 100644 --- a/tests/unit/interpreter.cpp +++ b/tests/unit/interpreter.cpp @@ -64,7 +64,7 @@ class InterpreterTest : public ::testing::Test { const std::string testSuiteCsv = "interpreter_csv"; std::filesystem::path data_directory = std::filesystem::temp_directory_path() / "MG_tests_unit_interpreter"; - InterpreterTest() {} + InterpreterTest() = default; memgraph::storage::Config config{ [&]() { @@ -334,7 +334,7 @@ TYPED_TEST(InterpreterTest, Bfs) { auto kNumUnreachableNodes = 1000; auto kNumUnreachableEdges = 100000; auto kResCoeff = 5; - const auto kReachable = "reachable"; + const auto *const kReachable = "reachable"; const auto kId = "id"; if (std::is_same::value) { diff --git a/tests/unit/kvstore.cpp b/tests/unit/kvstore.cpp index 8af05e482..48ab7dc82 100644 --- a/tests/unit/kvstore.cpp +++ b/tests/unit/kvstore.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -20,9 +20,9 @@ namespace fs = std::filesystem; class KVStore : public ::testing::Test { protected: - virtual void SetUp() { memgraph::utils::EnsureDir(test_folder_); } + void SetUp() override { memgraph::utils::EnsureDir(test_folder_); } - virtual void TearDown() { fs::remove_all(test_folder_); } + void TearDown() override { fs::remove_all(test_folder_); } fs::path test_folder_{fs::temp_directory_path() / ("unit_kvstore_test_" + std::to_string(static_cast(getpid())))}; diff --git a/tests/unit/mgp_kafka_c_api.cpp b/tests/unit/mgp_kafka_c_api.cpp index 91218dc75..828d52163 100644 --- a/tests/unit/mgp_kafka_c_api.cpp +++ b/tests/unit/mgp_kafka_c_api.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -113,7 +113,7 @@ class MgpApiTest : public ::testing::Test { using KafkaMessage = MockedRdKafkaMessage; MgpApiTest() { messages_.emplace(CreateMockedBatch()); } - ~MgpApiTest() { messages_.reset(); } + ~MgpApiTest() override { messages_.reset(); } mgp_messages &Messages() { return *messages_; } diff --git a/tests/unit/plan_pretty_print.cpp b/tests/unit/plan_pretty_print.cpp index 94b7342f9..97f1355cb 100644 --- a/tests/unit/plan_pretty_print.cpp +++ b/tests/unit/plan_pretty_print.cpp @@ -46,7 +46,7 @@ class PrintToJsonTest : public ::testing::Test { dba_storage(db->Access()), dba(dba_storage.get()) {} - ~PrintToJsonTest() { + ~PrintToJsonTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index e06d63908..a14ef2d30 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -46,9 +46,7 @@ #include "storage/v2/id_types.hpp" #include "utils/string.hpp" -namespace memgraph::query { - -namespace test_common { +namespace memgraph::query::test_common { auto ToIntList(const TypedValue &t) { std::vector list; @@ -189,7 +187,7 @@ auto GetEdgeVariable(AstStorage &storage, const std::string &name, EdgeAtom::Typ for (const auto &type : edge_types) { types.push_back(storage.GetEdgeTypeIx(type)); } - auto r_val = storage.Create(storage.Create(name), type, dir, types); + auto *r_val = storage.Create(storage.Create(name), type, dir, types); r_val->filter_lambda_.inner_edge = flambda_inner_edge ? flambda_inner_edge : storage.Create(memgraph::utils::RandomString(20)); @@ -215,14 +213,14 @@ auto GetEdgeVariable(AstStorage &storage, const std::string &name, EdgeAtom::Typ /// Name is used to create the Identifier which is assigned to the node. auto GetNode(AstStorage &storage, const std::string &name, std::optional label = std::nullopt, const bool user_declared = true) { - auto node = storage.Create(storage.Create(name, user_declared)); + auto *node = storage.Create(storage.Create(name, user_declared)); if (label) node->labels_.emplace_back(storage.GetLabelIx(*label)); return node; } /// Create a Pattern with given atoms. auto GetPattern(AstStorage &storage, std::vector atoms) { - auto pattern = storage.Create(); + auto *pattern = storage.Create(); pattern->identifier_ = storage.Create(memgraph::utils::RandomString(20), false); pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); return pattern; @@ -230,7 +228,7 @@ auto GetPattern(AstStorage &storage, std::vector atoms) { /// Create a Pattern with given name and atoms. auto GetPattern(AstStorage &storage, const std::string &name, std::vector atoms) { - auto pattern = storage.Create(); + auto *pattern = storage.Create(); pattern->identifier_ = storage.Create(name, true); pattern->atoms_.insert(pattern->atoms_.begin(), atoms.begin(), atoms.end()); return pattern; @@ -377,7 +375,7 @@ void FillReturnBody(AstStorage &storage, ReturnBody &body, const std::string &na /// @sa GetWith template auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { - auto ret = storage.Create(); + auto *ret = storage.Create(); ret->body_.distinct = distinct; FillReturnBody(storage, ret->body_, exprs...); return ret; @@ -390,7 +388,7 @@ auto GetReturn(AstStorage &storage, bool distinct, T... exprs) { /// @sa GetReturn template auto GetWith(AstStorage &storage, bool distinct, T... exprs) { - auto with = storage.Create(); + auto *with = storage.Create(); with->body_.distinct = distinct; FillReturnBody(storage, with->body_, exprs...); return with; @@ -407,7 +405,7 @@ auto GetUnwind(AstStorage &storage, Expression *expr, NamedExpression *as) { /// Create the delete clause with given named expressions. auto GetDelete(AstStorage &storage, std::vector exprs, bool detach = false) { - auto del = storage.Create(); + auto *del = storage.Create(); del->expressions_.insert(del->expressions_.begin(), exprs.begin(), exprs.end()); del->detach_ = detach; return del; @@ -495,9 +493,7 @@ auto GetForeach(AstStorage &storage, NamedExpression *named_expr, const std::vec return storage.Create(named_expr, clauses); } -} // namespace test_common - -} // namespace memgraph::query +} // namespace memgraph::query::test_common /// All the following macros implicitly pass `storage` variable to functions. /// You need to have `AstStorage storage;` somewhere in scope to use them. diff --git a/tests/unit/query_cost_estimator.cpp b/tests/unit/query_cost_estimator.cpp index a2c7b4a48..f3d6b3864 100644 --- a/tests/unit/query_cost_estimator.cpp +++ b/tests/unit/query_cost_estimator.cpp @@ -49,7 +49,7 @@ class QueryCostEstimator : public ::testing::Test { Parameters parameters_; int symbol_count = 0; - void SetUp() { + void SetUp() override { { auto unique_acc = db->UniqueAccess(); ASSERT_FALSE(unique_acc->CreateIndex(label).HasError()); @@ -131,7 +131,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelCardinality) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", const_val); EXPECT_COST(1 * CostParam::MakeScanAllByLabelPropertyValue); } @@ -139,7 +139,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstExpr) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", // once we make expression const-folding this test case will fail storage_.Create(const_val)); @@ -149,7 +149,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyValueConstExpr) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeUpperConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", nullopt, InclusiveBound(const_val)); // cardinality estimation is exact for very small indexes @@ -159,7 +159,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeUpperConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeLowerConstant) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(17), Parameter(17)}) { + for (auto *const_val : {Literal(17), Parameter(17)}) { MakeOp(nullptr, NextSymbol(), label, property, "property", InclusiveBound(const_val), nullopt); // cardinality estimation is exact for very small indexes @@ -169,7 +169,7 @@ TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeLowerConstant) { TEST_F(QueryCostEstimator, ScanAllByLabelPropertyRangeConstExpr) { AddVertices(100, 30, 20); - for (auto const_val : {Literal(12), Parameter(12)}) { + for (auto *const_val : {Literal(12), Parameter(12)}) { auto bound = std::make_optional( memgraph::utils::MakeBoundInclusive(static_cast(storage_.Create(const_val)))); MakeOp(nullptr, NextSymbol(), label, property, "property", bound, nullopt); diff --git a/tests/unit/query_dump.cpp b/tests/unit/query_dump.cpp index f212d9189..5556ab90a 100644 --- a/tests/unit/query_dump.cpp +++ b/tests/unit/query_dump.cpp @@ -46,7 +46,7 @@ const char *kRemoveInternalLabelProperty = "MATCH (u) REMOVE u:__mg_vertex__, u. struct DatabaseState { struct Vertex { int64_t id; - std::set labels; + std::set> labels; std::map props; }; @@ -67,7 +67,7 @@ struct DatabaseState { struct LabelPropertiesItem { std::string label; - std::set properties; + std::set> properties; }; std::set vertices; @@ -139,7 +139,7 @@ DatabaseState GetState(memgraph::storage::Storage *db) { std::set vertices; auto dba = db->Access(); for (const auto &vertex : dba->Vertices(memgraph::storage::View::NEW)) { - std::set labels; + std::set> labels; auto maybe_labels = vertex.Labels(memgraph::storage::View::NEW); MG_ASSERT(maybe_labels.HasValue()); for (const auto &label : *maybe_labels) { @@ -198,7 +198,7 @@ DatabaseState GetState(memgraph::storage::Storage *db) { existence_constraints.insert({dba->LabelToName(item.first), dba->PropertyToName(item.second)}); } for (const auto &item : info.unique) { - std::set properties; + std::set> properties; for (const auto &property : item.second) { properties.insert(dba->PropertyToName(property)); } diff --git a/tests/unit/query_expression_evaluator.cpp b/tests/unit/query_expression_evaluator.cpp index 86ac5d624..44d3ed301 100644 --- a/tests/unit/query_expression_evaluator.cpp +++ b/tests/unit/query_expression_evaluator.cpp @@ -69,7 +69,7 @@ class ExpressionEvaluatorTest : public ::testing::Test { storage_dba(db->Access()), dba(storage_dba.get()) {} - ~ExpressionEvaluatorTest() { + ~ExpressionEvaluatorTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } @@ -580,7 +580,7 @@ TYPED_TEST(ExpressionEvaluatorTest, VertexAndEdgeIndexing) { TYPED_TEST(ExpressionEvaluatorTest, TypedValueListIndexing) { auto list_vector = memgraph::utils::pmr::vector(this->ctx.memory); list_vector.emplace_back("string1"); - list_vector.emplace_back(TypedValue("string2")); + list_vector.emplace_back("string2"); auto *identifier = this->storage.template Create("n"); auto node_symbol = this->symbol_table.CreateSymbol("n", true); @@ -1196,7 +1196,7 @@ class ExpressionEvaluatorPropertyLookup : public ExpressionEvaluatorTeststorage.template Create("element"); Symbol symbol = this->symbol_table.CreateSymbol("element", true); - void SetUp() { identifier->MapTo(symbol); } + void SetUp() override { identifier->MapTo(symbol); } auto Value(std::pair property) { auto *op = this->storage.template Create(identifier, this->storage.GetPropertyIx(property.first)); @@ -1388,7 +1388,7 @@ class ExpressionEvaluatorAllPropertiesLookup : public ExpressionEvaluatorTeststorage.template Create("element"); Symbol symbol = this->symbol_table.CreateSymbol("element", true); - void SetUp() { identifier->MapTo(symbol); } + void SetUp() override { identifier->MapTo(symbol); } auto Value() { auto *op = this->storage.template Create(identifier); diff --git a/tests/unit/query_plan_checker.hpp b/tests/unit/query_plan_checker.hpp index 37695b581..6f2f23df7 100644 --- a/tests/unit/query_plan_checker.hpp +++ b/tests/unit/query_plan_checker.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "query/frontend/semantic/symbol_generator.hpp" #include "query/frontend/semantic/symbol_table.hpp" @@ -23,7 +24,7 @@ namespace memgraph::query::plan { class BaseOpChecker { public: - virtual ~BaseOpChecker() {} + virtual ~BaseOpChecker() = default; virtual void CheckOp(LogicalOperator &, const SymbolTable &) = 0; }; @@ -187,7 +188,7 @@ using ExpectEvaluatePatternFilter = OpChecker; class ExpectFilter : public OpChecker { public: - ExpectFilter(const std::vector> &pattern_filters = {}) + explicit ExpectFilter(const std::vector> &pattern_filters = {}) : pattern_filters_(pattern_filters) {} void ExpectOp(Filter &filter, const SymbolTable &symbol_table) override { @@ -222,7 +223,7 @@ class ExpectForeach : public OpChecker { class ExpectApply : public OpChecker { public: - ExpectApply(const std::list &subquery) : subquery_(subquery) {} + explicit ExpectApply(const std::list &subquery) : subquery_(subquery) {} void ExpectOp(Apply &apply, const SymbolTable &symbol_table) override { PlanChecker check_subquery(subquery_, symbol_table); @@ -287,7 +288,7 @@ class ExpectAggregate : public OpChecker { auto aggr_it = aggregations_.begin(); for (const auto &aggr_elem : op.aggregations_) { ASSERT_NE(aggr_it, aggregations_.end()); - auto aggr = *aggr_it++; + auto *aggr = *aggr_it++; // TODO: Proper expression equality EXPECT_EQ(typeid(aggr_elem.value).hash_code(), typeid(aggr->expression1_).hash_code()); EXPECT_EQ(typeid(aggr_elem.key).hash_code(), typeid(aggr->expression2_).hash_code()); @@ -472,9 +473,9 @@ class ExpectIndexedJoin : public OpChecker { class ExpectCallProcedure : public OpChecker { public: - ExpectCallProcedure(const std::string &name, const std::vector &args, + ExpectCallProcedure(std::string name, const std::vector &args, const std::vector &fields, const std::vector &result_syms) - : name_(name), args_(args), fields_(fields), result_syms_(result_syms) {} + : name_(std::move(name)), args_(args), fields_(fields), result_syms_(result_syms) {} void ExpectOp(CallProcedure &op, const SymbolTable &symbol_table) override { EXPECT_EQ(op.procedure_name_, name_); @@ -526,7 +527,7 @@ class FakeDbAccessor { } int64_t VerticesCount(memgraph::storage::LabelId label, memgraph::storage::PropertyId property) const { - for (auto &index : label_property_index_) { + for (const auto &index : label_property_index_) { if (std::get<0>(index) == label && std::get<1>(index) == property) { return std::get<2>(index); } @@ -539,7 +540,7 @@ class FakeDbAccessor { } bool LabelPropertyIndexExists(memgraph::storage::LabelId label, memgraph::storage::PropertyId property) const { - for (auto &index : label_property_index_) { + for (const auto &index : label_property_index_) { if (std::get<0>(index) == label && std::get<1>(index) == property) { return true; } diff --git a/tests/unit/query_plan_common.hpp b/tests/unit/query_plan_common.hpp index 4deba7648..1831a53d7 100644 --- a/tests/unit/query_plan_common.hpp +++ b/tests/unit/query_plan_common.hpp @@ -64,7 +64,7 @@ std::vector> CollectProduce(const Produce &produce, Exec // collect the symbols from the return clause std::vector symbols; - for (auto named_expression : produce.named_expressions_) + for (auto *named_expression : produce.named_expressions_) symbols.emplace_back(context->symbol_table.at(*named_expression)); // stream out results @@ -109,7 +109,7 @@ struct ScanAllTuple { ScanAllTuple MakeScanAll(AstStorage &storage, SymbolTable &symbol_table, const std::string &identifier, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, view); @@ -125,7 +125,7 @@ ScanAllTuple MakeScanAll(AstStorage &storage, SymbolTable &symbol_table, const s ScanAllTuple MakeScanAllByLabel(AstStorage &storage, SymbolTable &symbol_table, const std::string &identifier, memgraph::storage::LabelId label, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, label, view); @@ -144,7 +144,7 @@ ScanAllTuple MakeScanAllByLabelPropertyRange(AstStorage &storage, SymbolTable &s std::optional upper_bound, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = std::make_shared(input, symbol, label, property, property_name, @@ -163,7 +163,7 @@ ScanAllTuple MakeScanAllByLabelPropertyValue(AstStorage &storage, SymbolTable &s const std::string &property_name, Expression *value, std::shared_ptr input = {nullptr}, memgraph::storage::View view = memgraph::storage::View::OLD) { - auto node = memgraph::query::test_common::GetNode(storage, identifier); + auto *node = memgraph::query::test_common::GetNode(storage, identifier); auto symbol = symbol_table.CreateSymbol(identifier, true); node->identifier_->MapTo(symbol); auto logical_op = @@ -183,11 +183,11 @@ ExpandTuple MakeExpand(AstStorage &storage, SymbolTable &symbol_table, std::shar Symbol input_symbol, const std::string &edge_identifier, EdgeAtom::Direction direction, const std::vector &edge_types, const std::string &node_identifier, bool existing_node, memgraph::storage::View view) { - auto edge = memgraph::query::test_common::GetEdge(storage, edge_identifier, direction); + auto *edge = memgraph::query::test_common::GetEdge(storage, edge_identifier, direction); auto edge_sym = symbol_table.CreateSymbol(edge_identifier, true); edge->identifier_->MapTo(edge_sym); - auto node = memgraph::query::test_common::GetNode(storage, node_identifier); + auto *node = memgraph::query::test_common::GetNode(storage, node_identifier); auto node_sym = symbol_table.CreateSymbol(node_identifier, true); node->identifier_->MapTo(node_sym); diff --git a/tests/unit/query_plan_edge_cases.cpp b/tests/unit/query_plan_edge_cases.cpp index 52769fa55..d0953651e 100644 --- a/tests/unit/query_plan_edge_cases.cpp +++ b/tests/unit/query_plan_edge_cases.cpp @@ -43,7 +43,7 @@ class QueryExecution : public testing::Test { std::optional repl_state; std::optional> db_gk; - void SetUp() { + void SetUp() override { auto config = [&]() { memgraph::storage::Config config{}; config.durability.storage_directory = data_directory; @@ -70,7 +70,7 @@ class QueryExecution : public testing::Test { interpreter_.emplace(&*interpreter_context_, *db_acc_); } - void TearDown() { + void TearDown() override { interpreter_ = std::nullopt; interpreter_context_ = std::nullopt; db_acc_.reset(); diff --git a/tests/unit/query_plan_operator_to_string.cpp b/tests/unit/query_plan_operator_to_string.cpp index 36918425d..27d27f0d1 100644 --- a/tests/unit/query_plan_operator_to_string.cpp +++ b/tests/unit/query_plan_operator_to_string.cpp @@ -40,7 +40,7 @@ class OperatorToStringTest : public ::testing::Test { dba_storage(db->Access()), dba(dba_storage.get()) {} - ~OperatorToStringTest() { + ~OperatorToStringTest() override { if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } diff --git a/tests/unit/query_required_privileges.cpp b/tests/unit/query_required_privileges.cpp index e43525f5b..630bc44a0 100644 --- a/tests/unit/query_required_privileges.cpp +++ b/tests/unit/query_required_privileges.cpp @@ -153,7 +153,7 @@ TEST_F(TestPrivilegeExtractor, DumpDatabase) { } TEST_F(TestPrivilegeExtractor, ReadFile) { - auto load_csv = storage.Create(); + auto *load_csv = storage.Create(); load_csv->row_var_ = IDENT("row"); auto *query = QUERY(SINGLE_QUERY(load_csv)); EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::READ_FILE)); diff --git a/tests/unit/query_semantic.cpp b/tests/unit/query_semantic.cpp index d49eb3798..5d015f103 100644 --- a/tests/unit/query_semantic.cpp +++ b/tests/unit/query_semantic.cpp @@ -54,17 +54,17 @@ TYPED_TEST(TestSymbolGenerator, MatchNodeReturn) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for pattern, node_atom_1 and named_expr in return EXPECT_EQ(symbol_table.max_position(), 3); - auto match = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = match->patterns_[0]; + auto *match = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = match->patterns_[0]; auto pattern_sym = symbol_table.at(*pattern->identifier_); EXPECT_EQ(pattern_sym.type(), Symbol::Type::PATH); EXPECT_FALSE(pattern_sym.user_declared()); - auto node_atom = dynamic_cast(pattern->atoms_[0]); + auto *node_atom = dynamic_cast(pattern->atoms_[0]); auto node_sym = symbol_table.at(*node_atom->identifier_); EXPECT_EQ(node_sym.name(), "node_atom_1"); EXPECT_EQ(node_sym.type(), Symbol::Type::VERTEX); - auto ret = dynamic_cast(query_ast->single_query_->clauses_[1]); - auto named_expr = ret->body_.named_expressions[0]; + auto *ret = dynamic_cast(query_ast->single_query_->clauses_[1]); + auto *named_expr = ret->body_.named_expressions[0]; auto column_sym = symbol_table.at(*named_expr); EXPECT_EQ(node_sym.name(), column_sym.name()); EXPECT_NE(node_sym, column_sym); @@ -78,8 +78,8 @@ TYPED_TEST(TestSymbolGenerator, MatchNamedPattern) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for p, node_atom_1 and named_expr in return EXPECT_EQ(symbol_table.max_position(), 3); - auto match = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = match->patterns_[0]; + auto *match = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = match->patterns_[0]; auto pattern_sym = symbol_table.at(*pattern->identifier_); EXPECT_EQ(pattern_sym.type(), Symbol::Type::PATH); EXPECT_EQ(pattern_sym.name(), "p"); @@ -114,14 +114,14 @@ TYPED_TEST(TestSymbolGenerator, CreateNodeReturn) { auto symbol_table = memgraph::query::MakeSymbolTable(query_ast); // symbols for pattern, `n` and named_expr EXPECT_EQ(symbol_table.max_position(), 3); - auto create = dynamic_cast(query_ast->single_query_->clauses_[0]); - auto pattern = create->patterns_[0]; - auto node_atom = dynamic_cast(pattern->atoms_[0]); + auto *create = dynamic_cast(query_ast->single_query_->clauses_[0]); + auto *pattern = create->patterns_[0]; + auto *node_atom = dynamic_cast(pattern->atoms_[0]); auto node_sym = symbol_table.at(*node_atom->identifier_); EXPECT_EQ(node_sym.name(), "n"); EXPECT_EQ(node_sym.type(), Symbol::Type::VERTEX); - auto ret = dynamic_cast(query_ast->single_query_->clauses_[1]); - auto named_expr = ret->body_.named_expressions[0]; + auto *ret = dynamic_cast(query_ast->single_query_->clauses_[1]); + auto *named_expr = ret->body_.named_expressions[0]; auto column_sym = symbol_table.at(*named_expr); EXPECT_EQ(node_sym.name(), column_sym.name()); EXPECT_NE(node_sym, column_sym); @@ -151,7 +151,7 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateRedeclareNode) { TYPED_TEST(TestSymbolGenerator, MatchCreateRedeclareEdge) { // AST with redeclaring a match edge variable in create: // MATCH (n) -[r]- (m) CREATE (n) -[r :relationship]-> (l) - auto relationship = "relationship"; + const auto *relationship = "relationship"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::OUT, {relationship}), NODE("l"))))); @@ -176,8 +176,8 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateTypeMismatch) { TYPED_TEST(TestSymbolGenerator, CreateMultipleEdgeType) { // Multiple edge relationship are not allowed when creating edges. // CREATE (n) -[r :rel1 | :rel2]-> (m) - auto rel1 = "rel1"; - auto rel2 = "rel2"; + const auto *rel1 = "rel1"; + const auto *rel2 = "rel2"; auto edge = EDGE("r", EdgeAtom::Direction::OUT, {rel1}); edge->edge_types_.emplace_back(this->storage.GetEdgeTypeIx(rel2)); auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), edge, NODE("m"))))); @@ -187,7 +187,7 @@ TYPED_TEST(TestSymbolGenerator, CreateMultipleEdgeType) { TYPED_TEST(TestSymbolGenerator, CreateBidirectionalEdge) { // Bidirectional relationships are not allowed when creating edges. // CREATE (n) -[r :rel1]- (m) - auto rel1 = "rel1"; + const auto *rel1 = "rel1"; auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::BOTH, {rel1}), NODE("m"))))); EXPECT_THROW(memgraph::query::MakeSymbolTable(query), SemanticException); } @@ -276,8 +276,8 @@ TYPED_TEST(TestSymbolGenerator, MatchWithWhereUnbound) { TYPED_TEST(TestSymbolGenerator, CreateMultiExpand) { // Test CREATE (n) -[r :r]-> (m), (n) - [p :p]-> (l) - auto r_type = "r"; - auto p_type = "p"; + const auto *r_type = "r"; + const auto *p_type = "p"; auto node_n1 = NODE("n"); auto edge_r = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); auto node_m = NODE("m"); @@ -308,8 +308,8 @@ TYPED_TEST(TestSymbolGenerator, CreateMultiExpand) { TYPED_TEST(TestSymbolGenerator, MatchCreateExpandLabel) { // Test MATCH (n) CREATE (m) -[r :r]-> (n:label) - auto r_type = "r"; - auto label = "label"; + const auto *r_type = "r"; + const auto *label = "label"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), CREATE(PATTERN(NODE("m"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), NODE("n", label))))); @@ -318,7 +318,7 @@ TYPED_TEST(TestSymbolGenerator, MatchCreateExpandLabel) { TYPED_TEST(TestSymbolGenerator, CreateExpandProperty) { // Test CREATE (n) -[r :r]-> (n {prop: 42}) - auto r_type = "r"; + const auto *r_type = "r"; auto n_prop = NODE("n"); std::get<0>(n_prop->properties_)[this->storage.GetPropertyIx("prop")] = LITERAL(42); auto query = QUERY(SINGLE_QUERY(CREATE(PATTERN(NODE("n"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), n_prop)))); @@ -379,7 +379,7 @@ TYPED_TEST(TestSymbolGenerator, MatchPropCreateNodeProp) { TYPED_TEST(TestSymbolGenerator, CreateNodeEdge) { // Test CREATE (n), (n) -[r :r]-> (n) - auto r_type = "r"; + const auto *r_type = "r"; auto node_1 = NODE("n"); auto node_2 = NODE("n"); auto edge = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); @@ -396,7 +396,7 @@ TYPED_TEST(TestSymbolGenerator, CreateNodeEdge) { TYPED_TEST(TestSymbolGenerator, MatchWithCreate) { // Test MATCH (n) WITH n AS m CREATE (m) -[r :r]-> (m) - auto r_type = "r"; + const auto *r_type = "r"; auto node_1 = NODE("n"); auto node_2 = NODE("m"); auto edge = EDGE("r", EdgeAtom::Direction::OUT, {r_type}); @@ -500,7 +500,7 @@ TYPED_TEST(TestSymbolGenerator, MergeVariableError) { TYPED_TEST(TestSymbolGenerator, MergeVariableErrorEdge) { // Test MATCH (n) -[r]- (m) MERGE (a) -[r :rel]- (b) - auto rel = "rel"; + const auto *rel = "rel"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"), EDGE("r"), NODE("m"))), MERGE(PATTERN(NODE("a"), EDGE("r", EdgeAtom::Direction::BOTH, {rel}), NODE("b"))))); EXPECT_THROW(memgraph::query::MakeSymbolTable(query), RedeclareVariableError); @@ -516,7 +516,7 @@ TYPED_TEST(TestSymbolGenerator, MergeEdgeWithoutType) { TYPED_TEST(TestSymbolGenerator, MergeOnMatchOnCreate) { // Test MATCH (n) MERGE (n) -[r :rel]- (m) ON MATCH SET n.prop = 42 // ON CREATE SET m.prop = 42 RETURN r AS r - auto rel = "rel"; + const auto *rel = "rel"; auto prop = this->dba.NameToProperty("prop"); auto match_n = NODE("n"); auto merge_n = NODE("n"); @@ -641,8 +641,8 @@ TYPED_TEST(TestSymbolGenerator, MatchReturnAsteriskNoUserVariables) { TYPED_TEST(TestSymbolGenerator, MatchMergeExpandLabel) { // Test MATCH (n) MERGE (m) -[r :r]-> (n:label) - auto r_type = "r"; - auto label = "label"; + const auto *r_type = "r"; + const auto *label = "label"; auto query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), MERGE(PATTERN(NODE("m"), EDGE("r", EdgeAtom::Direction::OUT, {r_type}), NODE("n", label))))); diff --git a/tests/unit/rpc_messages.hpp b/tests/unit/rpc_messages.hpp index 6058c37cf..2f8175a36 100644 --- a/tests/unit/rpc_messages.hpp +++ b/tests/unit/rpc_messages.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "rpc/messages.hpp" #include "slk/serialization.hpp" #include "utils/typeinfo.hpp" @@ -18,7 +20,7 @@ struct SumReq { static const memgraph::utils::TypeInfo kType; - SumReq() {} // Needed for serialization. + SumReq() = default; // Needed for serialization. SumReq(int x, int y) : x(x), y(y) {} static void Load(SumReq *obj, memgraph::slk::Reader *reader); @@ -33,8 +35,8 @@ const memgraph::utils::TypeInfo SumReq::kType{memgraph::utils::TypeId::UNKNOWN, struct SumRes { static const memgraph::utils::TypeInfo kType; - SumRes() {} // Needed for serialization. - SumRes(int sum) : sum(sum) {} + SumRes() = default; // Needed for serialization. + explicit SumRes(int sum) : sum(sum) {} static void Load(SumRes *obj, memgraph::slk::Reader *reader); static void Save(const SumRes &obj, memgraph::slk::Builder *builder); @@ -57,8 +59,8 @@ using Sum = memgraph::rpc::RequestResponse; struct EchoMessage { static const memgraph::utils::TypeInfo kType; - EchoMessage() {} // Needed for serialization. - EchoMessage(const std::string &data) : data(data) {} + EchoMessage() = default; // Needed for serialization. + explicit EchoMessage(std::string data) : data(std::move(data)) {} static void Load(EchoMessage *obj, memgraph::slk::Reader *reader); static void Save(const EchoMessage &obj, memgraph::slk::Builder *builder); diff --git a/tests/unit/slk_core.cpp b/tests/unit/slk_core.cpp index 65b916db3..9b63b82f8 100644 --- a/tests/unit/slk_core.cpp +++ b/tests/unit/slk_core.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -108,7 +108,7 @@ TEST(SlkCore, SetPrimitive) { } TEST(SlkCore, SetString) { - std::set original{"hai hai hai", "nandare!"}; + std::set> original{"hai hai hai", "nandare!"}; memgraph::slk::Loopback loopback; auto builder = loopback.GetBuilder(); memgraph::slk::Save(original, builder); @@ -116,7 +116,7 @@ TEST(SlkCore, SetString) { for (const auto &item : original) { size += sizeof(uint64_t) + item.size(); } - std::set decoded; + std::set> decoded; auto reader = loopback.GetReader(); memgraph::slk::Load(&decoded, reader); ASSERT_EQ(original, decoded); diff --git a/tests/unit/slk_streams.cpp b/tests/unit/slk_streams.cpp index 634eaecb5..8a673af1c 100644 --- a/tests/unit/slk_streams.cpp +++ b/tests/unit/slk_streams.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -66,7 +66,7 @@ std::vector BufferToBinaryData(const uint8_t *data, size_t size, std size_t pos = 0; for (size_t i = 0; i < sizes.size(); ++i) { EXPECT_GE(size, pos + sizes[i]); - ret.push_back({data + pos, sizes[i]}); + ret.emplace_back(data + pos, sizes[i]); pos += sizes[i]; } return ret; diff --git a/tests/unit/storage_rocks.cpp b/tests/unit/storage_rocks.cpp index 42890c383..6d5db7d75 100644 --- a/tests/unit/storage_rocks.cpp +++ b/tests/unit/storage_rocks.cpp @@ -49,7 +49,7 @@ class RocksDBStorageTest : public ::testing::TestWithParam { disk_test_utils::RemoveRocksDbDirs(testSuite); } - ~RocksDBStorageTest() override {} + ~RocksDBStorageTest() override = default; protected: std::unique_ptr storage; diff --git a/tests/unit/storage_v2_property_store.cpp b/tests/unit/storage_v2_property_store.cpp index 9da503f71..59b38c632 100644 --- a/tests/unit/storage_v2_property_store.cpp +++ b/tests/unit/storage_v2_property_store.cpp @@ -670,7 +670,7 @@ TEST(PropertyStore, SetMultipleProperties) { const std::map data_in_map{data.begin(), data.end()}; auto check_store = [data](const memgraph::storage::PropertyStore &store) { - for (auto &[key, value] : data) { + for (const auto &[key, value] : data) { ASSERT_TRUE(store.IsPropertyEqual(key, value)); } }; diff --git a/tests/unit/storage_v2_wal_file.cpp b/tests/unit/storage_v2_wal_file.cpp index 19b633f9f..a67b09305 100644 --- a/tests/unit/storage_v2_wal_file.cpp +++ b/tests/unit/storage_v2_wal_file.cpp @@ -231,7 +231,7 @@ class DeltaGenerator final { } void AppendOperation(memgraph::storage::durability::StorageMetadataOperation operation, const std::string &label, - const std::set properties = {}, const std::string &stats = {}) { + const std::set> properties = {}, const std::string &stats = {}) { auto label_id = memgraph::storage::LabelId::FromUint(mapper_.NameToId(label)); std::set property_ids; for (const auto &property : properties) { @@ -378,7 +378,7 @@ void AssertWalDataEqual(const DeltaGenerator::DataT &data, const std::filesystem class WalFileTest : public ::testing::TestWithParam { public: - WalFileTest() {} + WalFileTest() = default; void SetUp() override { Clear(); } @@ -710,7 +710,7 @@ TEST_P(WalFileTest, PartialData) { class StorageModeWalFileTest : public ::testing::TestWithParam { public: - StorageModeWalFileTest() {} + StorageModeWalFileTest() = default; void SetUp() override { Clear(); } diff --git a/tests/unit/utils_file.cpp b/tests/unit/utils_file.cpp index 05d8184be..89c3699a9 100644 --- a/tests/unit/utils_file.cpp +++ b/tests/unit/utils_file.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -132,7 +132,7 @@ class UtilsFileTest : public ::testing::Test { private: void Clear() { if (fs::exists(storage)) { - for (auto &file : fs::recursive_directory_iterator(storage)) { + for (const auto &file : fs::recursive_directory_iterator(storage)) { std::error_code error_code; // For exception suppression. fs::permissions(file.path(), fs::perms::owner_all, fs::perm_options::add, error_code); } diff --git a/tests/unit/utils_string.cpp b/tests/unit/utils_string.cpp index 4cde1dfe1..cefe57a6a 100644 --- a/tests/unit/utils_string.cpp +++ b/tests/unit/utils_string.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -151,7 +151,7 @@ TEST(String, RandomString) { EXPECT_EQ(RandomString(1).size(), 1); EXPECT_EQ(RandomString(42).size(), 42); - std::set string_set; + std::set> string_set; for (int i = 0; i < 20; ++i) string_set.emplace(RandomString(256)); EXPECT_EQ(string_set.size(), 20); From 8b9e1fa08bd2094bf89313e94fe31028d28f4ff1 Mon Sep 17 00:00:00 2001 From: andrejtonev <29177572+andrejtonev@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:02:35 +0100 Subject: [PATCH 09/10] Replication refactor part 6 (#1484) Single (instance level) connection to a replica (messages from all databases get multiplexed through it) ReplicationClient split in two: ReplicationClient and ReplicationStorageClient New ReplicationClient, moved under replication, handles the raw connection, owned by MainRoleData ReplicationStorageClient handles the storage <-> replica state machine and holds to a stream Removed epoch and storage from *Clients rpc::Stream proactively aborts on error and sets itself to a defunct state Removed HandleRpcFailure, instead we simply log the error and let the FrequentCheck handle re-connection replica_state is now a synced variable ReplicaStorageClient state machine bugfixes Single FrequentCheck that goes through DBMS Moved ReplicationState under DbmsHandler Moved some replication startup logic under the DbmsHandler's constructor Removed InMemoryReplicationClient CreateReplicationClient has been removed from Storage Simplified GetRecoverySteps and made safer --------- Co-authored-by: Gareth Lloyd --- release/get_version.py | 14 +- src/dbms/CMakeLists.txt | 2 +- src/dbms/constants.hpp | 6 + src/dbms/database.cpp | 2 +- src/dbms/database.hpp | 2 +- src/dbms/database_handler.hpp | 3 +- src/dbms/dbms_handler.cpp | 75 ++++ src/dbms/dbms_handler.hpp | 69 ++-- src/dbms/inmemory/storage_helper.hpp | 10 +- src/dbms/replication_client.cpp | 34 ++ src/dbms/replication_client.hpp | 21 ++ src/dbms/replication_handler.cpp | 164 ++++---- src/dbms/replication_handler.hpp | 6 +- src/memgraph.cpp | 25 +- src/query/frontend/ast/ast.hpp | 2 +- src/query/interpreter.cpp | 55 ++- src/replication/CMakeLists.txt | 4 + .../include/replication/messages.hpp | 44 +++ .../replication/replication_client.hpp | 82 ++++ .../replication/replication_server.hpp | 24 -- src/replication/include/replication/state.hpp | 28 +- src/replication/messages.cpp | 65 ++++ src/replication/replication_client.cpp | 40 ++ src/replication/replication_server.cpp | 46 +-- src/replication/state.cpp | 39 +- src/rpc/client.hpp | 57 ++- src/slk/streams.cpp | 8 +- src/slk/streams.hpp | 8 +- src/storage/v2/CMakeLists.txt | 3 +- src/storage/v2/disk/storage.hpp | 6 - src/storage/v2/durability/durability.cpp | 10 +- .../v2/inmemory/replication/recovery.cpp | 238 ++++++++++++ .../v2/inmemory/replication/recovery.hpp | 32 ++ .../replication/replication_client.cpp | 349 ------------------ .../replication/replication_client.hpp | 38 -- src/storage/v2/inmemory/storage.cpp | 16 +- src/storage/v2/inmemory/storage.hpp | 12 +- src/storage/v2/replication/enums.hpp | 2 +- src/storage/v2/replication/recovery.hpp | 28 ++ .../v2/replication/replication_client.cpp | 349 +++++++++--------- .../v2/replication/replication_client.hpp | 124 ++++--- .../replication/replication_storage_state.cpp | 18 +- .../replication/replication_storage_state.hpp | 25 +- src/storage/v2/storage.hpp | 10 +- .../show_while_creating_invalid_state.py | 155 +++++++- tests/integration/telemetry/client.cpp | 2 +- tests/unit/dbms_handler.cpp | 5 +- tests/unit/dbms_handler_community.cpp | 5 +- tests/unit/storage_v2_replication.cpp | 27 +- 49 files changed, 1398 insertions(+), 991 deletions(-) create mode 100644 src/dbms/dbms_handler.cpp create mode 100644 src/dbms/replication_client.cpp create mode 100644 src/dbms/replication_client.hpp create mode 100644 src/replication/include/replication/messages.hpp create mode 100644 src/replication/include/replication/replication_client.hpp create mode 100644 src/replication/messages.cpp create mode 100644 src/replication/replication_client.cpp create mode 100644 src/storage/v2/inmemory/replication/recovery.cpp create mode 100644 src/storage/v2/inmemory/replication/recovery.hpp delete mode 100644 src/storage/v2/inmemory/replication/replication_client.cpp delete mode 100644 src/storage/v2/inmemory/replication/replication_client.hpp create mode 100644 src/storage/v2/replication/recovery.hpp diff --git a/release/get_version.py b/release/get_version.py index cfce88475..a8539fab4 100755 --- a/release/get_version.py +++ b/release/get_version.py @@ -104,7 +104,9 @@ def retry(retry_limit, timeout=100): except Exception: time.sleep(timeout) return func(*args, **kwargs) + return wrapper + return inner_func @@ -200,19 +202,19 @@ if args.version: try: current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD") if current_branch != "master": - branches = get_output("git", "branch") - if "master" in branches: + branches = get_output("git", "branch", "-r", "--list", "origin/master") + if "origin/master" in branches: # If master is present locally, the fetch is allowed to fail # because this script will still be able to compare against the # master branch. try: - get_output("git", "fetch", "origin", "master:master") + get_output("git", "fetch", "origin", "master") except Exception: pass else: # If master is not present locally, the fetch command has to # succeed because something else will fail otherwise. - get_output("git", "fetch", "origin", "master:master") + get_output("git", "fetch", "origin", "master") except Exception: print("Fatal error while ensuring local master branch.") sys.exit(1) @@ -232,7 +234,7 @@ for branch in branches: match = branch_regex.match(branch) if match is not None: version = tuple(map(int, match.group(1).split("."))) - master_branch_merge = get_output("git", "merge-base", "master", branch) + master_branch_merge = get_output("git", "merge-base", "origin/master", branch) versions.append((version, branch, master_branch_merge)) versions.sort(reverse=True) @@ -243,7 +245,7 @@ current_version = None for version in versions: version_tuple, branch, master_branch_merge = version current_branch_merge = get_output("git", "merge-base", current_hash, branch) - master_current_merge = get_output("git", "merge-base", current_hash, "master") + master_current_merge = get_output("git", "merge-base", current_hash, "origin/master") # The first check checks whether this commit is a child of `master` and # the version branch was created before us. # The second check checks whether this commit is a child of the version diff --git a/src/dbms/CMakeLists.txt b/src/dbms/CMakeLists.txt index 8ec1e0972..f1df4985a 100644 --- a/src/dbms/CMakeLists.txt +++ b/src/dbms/CMakeLists.txt @@ -1,3 +1,3 @@ -add_library(mg-dbms STATIC database.cpp replication_handler.cpp inmemory/replication_handlers.cpp) +add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp) target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query) diff --git a/src/dbms/constants.hpp b/src/dbms/constants.hpp index 3ca61056b..e7ea9987b 100644 --- a/src/dbms/constants.hpp +++ b/src/dbms/constants.hpp @@ -15,4 +15,10 @@ namespace memgraph::dbms { constexpr static const char *kDefaultDB = "memgraph"; //!< Name of the default database +#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY +constexpr bool allow_mt_repl = true; +#else +constexpr bool allow_mt_repl = false; +#endif + } // namespace memgraph::dbms diff --git a/src/dbms/database.cpp b/src/dbms/database.cpp index 411e282e8..74ee13892 100644 --- a/src/dbms/database.cpp +++ b/src/dbms/database.cpp @@ -21,7 +21,7 @@ template struct memgraph::utils::Gatekeeper; namespace memgraph::dbms { -Database::Database(storage::Config config, const replication::ReplicationState &repl_state) +Database::Database(storage::Config config, replication::ReplicationState &repl_state) : trigger_store_(config.durability.storage_directory / "triggers"), streams_{config.durability.storage_directory / "streams"}, plan_cache_{FLAGS_query_plan_cache_max_size}, diff --git a/src/dbms/database.hpp b/src/dbms/database.hpp index 457aa1c1d..416ff76bc 100644 --- a/src/dbms/database.hpp +++ b/src/dbms/database.hpp @@ -48,7 +48,7 @@ class Database { * * @param config storage configuration */ - explicit Database(storage::Config config, const replication::ReplicationState &repl_state); + explicit Database(storage::Config config, replication::ReplicationState &repl_state); /** * @brief Returns the raw storage pointer. diff --git a/src/dbms/database_handler.hpp b/src/dbms/database_handler.hpp index a6b3b563b..617e614c3 100644 --- a/src/dbms/database_handler.hpp +++ b/src/dbms/database_handler.hpp @@ -51,8 +51,7 @@ class DatabaseHandler : public Handler { * @param config Storage configuration * @return HandlerT::NewResult */ - HandlerT::NewResult New(std::string_view name, storage::Config config, - const replication::ReplicationState &repl_state) { + HandlerT::NewResult New(std::string_view name, storage::Config config, replication::ReplicationState &repl_state) { // Control that no one is using the same data directory if (std::any_of(begin(), end(), [&](auto &elem) { auto db_acc = elem.second.access(); diff --git a/src/dbms/dbms_handler.cpp b/src/dbms/dbms_handler.cpp new file mode 100644 index 000000000..0af9364bf --- /dev/null +++ b/src/dbms/dbms_handler.cpp @@ -0,0 +1,75 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "dbms/dbms_handler.hpp" + +namespace memgraph::dbms { +#ifdef MG_ENTERPRISE +DbmsHandler::DbmsHandler( + storage::Config config, + memgraph::utils::Synchronized *auth, + bool recovery_on_startup, bool delete_on_drop) + : default_config_{std::move(config)}, + delete_on_drop_(delete_on_drop), + repl_state_{ReplicationStateRootPath(default_config_)} { + // TODO: Decouple storage config from dbms config + // TODO: Save individual db configs inside the kvstore and restore from there + storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases"); + const auto &db_dir = default_config_.durability.storage_directory; + const auto durability_dir = db_dir / ".durability"; + utils::EnsureDirOrDie(db_dir); + utils::EnsureDirOrDie(durability_dir); + durability_ = std::make_unique(durability_dir); + + // Generate the default database + MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB."); + + // Recover previous databases + if (recovery_on_startup) { + for (const auto &[name, _] : *durability_) { + if (name == kDefaultDB) continue; // Already set + spdlog::info("Restoring database {}.", name); + MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name); + spdlog::info("Database {} restored.", name); + } + } else { // Clear databases from the durability list and auth + auto locked_auth = auth->Lock(); + for (const auto &[name, _] : *durability_) { + if (name == kDefaultDB) continue; + locked_auth->DeleteDatabase(name); + durability_->Delete(name); + } + } + + // Startup replication state (if recovered at startup) + auto replica = [this](replication::RoleReplicaData const &data) { + // Register handlers + InMemoryReplicationHandlers::Register(this, *data.server); + if (!data.server->Start()) { + spdlog::error("Unable to start the replication server."); + return false; + } + return true; + }; + // Replication frequent check start + auto main = [this](replication::RoleMainData &data) { + for (auto &client : data.registered_replicas_) { + StartReplicaClient(*this, client); + } + return true; + }; + // Startup proccess for main/replica + MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()), + "Replica recovery failure!"); +} +#endif + +} // namespace memgraph::dbms diff --git a/src/dbms/dbms_handler.hpp b/src/dbms/dbms_handler.hpp index 27ab963d4..3151398ab 100644 --- a/src/dbms/dbms_handler.hpp +++ b/src/dbms/dbms_handler.hpp @@ -26,9 +26,11 @@ #include "auth/auth.hpp" #include "constants.hpp" #include "dbms/database.hpp" +#include "dbms/inmemory/replication_handlers.hpp" #ifdef MG_ENTERPRISE #include "dbms/database_handler.hpp" #endif +#include "dbms/replication_client.hpp" #include "global.hpp" #include "query/config.hpp" #include "query/interpreter_context.hpp" @@ -102,52 +104,22 @@ class DbmsHandler { * @param recovery_on_startup restore databases (and its content) and authentication data * @param delete_on_drop when dropping delete any associated directories on disk */ - DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state, auto *auth, - bool recovery_on_startup, bool delete_on_drop) - : lock_{utils::RWLock::Priority::READ}, - default_config_{std::move(config)}, - repl_state_(repl_state), - delete_on_drop_(delete_on_drop) { - // TODO: Decouple storage config from dbms config - // TODO: Save individual db configs inside the kvstore and restore from there - storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases"); - const auto &db_dir = default_config_.durability.storage_directory; - const auto durability_dir = db_dir / ".durability"; - utils::EnsureDirOrDie(db_dir); - utils::EnsureDirOrDie(durability_dir); - durability_ = std::make_unique(durability_dir); - - // Generate the default database - MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB."); - // Recover previous databases - if (recovery_on_startup) { - for (const auto &[name, _] : *durability_) { - if (name == kDefaultDB) continue; // Already set - spdlog::info("Restoring database {}.", name); - MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name); - spdlog::info("Database {} restored.", name); - } - } else { // Clear databases from the durability list and auth - auto locked_auth = auth->Lock(); - for (const auto &[name, _] : *durability_) { - if (name == kDefaultDB) continue; - locked_auth->DeleteDatabase(name); - durability_->Delete(name); - } - } - } + DbmsHandler(storage::Config config, + memgraph::utils::Synchronized *auth, + bool recovery_on_startup, bool delete_on_drop); // TODO If more arguments are added use a config strut #else /** * @brief Initialize the handler. A single database is supported in community edition. * * @param configs storage configuration */ - DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state) - : db_gatekeeper_{[&] { + DbmsHandler(storage::Config config) + : repl_state_{ReplicationStateRootPath(config)}, + db_gatekeeper_{[&] { config.name = kDefaultDB; return std::move(config); }(), - repl_state} {} + repl_state_} {} #endif #ifdef MG_ENTERPRISE @@ -248,6 +220,12 @@ class DbmsHandler { #endif } + replication::ReplicationState &ReplicationState() { return repl_state_; } + replication::ReplicationState const &ReplicationState() const { return repl_state_; } + + bool IsMain() const { return repl_state_.IsMain(); } + bool IsReplica() const { return repl_state_.IsReplica(); } + /** * @brief Return the statistics all databases. * @@ -536,14 +514,15 @@ class DbmsHandler { throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name); } - mutable LockT lock_; //!< protective lock - storage::Config default_config_; //!< Storage configuration used when creating new databases - const replication::ReplicationState &repl_state_; //!< Global replication state - DatabaseHandler db_handler_; //!< multi-tenancy storage handler - std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) - bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory - std::set> defunct_dbs_; //!< Databases that are in an unknown state due to various failures -#else + mutable LockT lock_{utils::RWLock::Priority::READ}; //!< protective lock + storage::Config default_config_; //!< Storage configuration used when creating new databases + DatabaseHandler db_handler_; //!< multi-tenancy storage handler + std::unique_ptr durability_; //!< list of active dbs (pointer so we can postpone its creation) + bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory + std::set defunct_dbs_; //!< Databases that are in an unknown state due to various failures +#endif + replication::ReplicationState repl_state_; //!< Global replication state +#ifndef MG_ENTERPRISE mutable utils::Gatekeeper db_gatekeeper_; //!< Single databases gatekeeper #endif }; diff --git a/src/dbms/inmemory/storage_helper.hpp b/src/dbms/inmemory/storage_helper.hpp index 347c16928..1cd9f9f4e 100644 --- a/src/dbms/inmemory/storage_helper.hpp +++ b/src/dbms/inmemory/storage_helper.hpp @@ -22,14 +22,8 @@ namespace memgraph::dbms { -#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY -constexpr bool allow_mt_repl = true; -#else -constexpr bool allow_mt_repl = false; -#endif - -inline std::unique_ptr CreateInMemoryStorage( - storage::Config config, const ::memgraph::replication::ReplicationState &repl_state) { +inline std::unique_ptr CreateInMemoryStorage(storage::Config config, + ::memgraph::replication::ReplicationState &repl_state) { const auto wal_mode = config.durability.snapshot_wal_mode; const auto name = config.name; auto storage = std::make_unique(std::move(config)); diff --git a/src/dbms/replication_client.cpp b/src/dbms/replication_client.cpp new file mode 100644 index 000000000..bfa4c622f --- /dev/null +++ b/src/dbms/replication_client.cpp @@ -0,0 +1,34 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "dbms/replication_client.hpp" + +namespace memgraph::dbms { + +void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client) { + // No client error, start instance level client + auto const &endpoint = client.rpc_client_.Endpoint(); + spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); + client.StartFrequentCheck([&dbms_handler](std::string_view name) { + // Working connection, check if any database has been left behind + dbms_handler.ForEach([name](dbms::Database *db) { + // Specific database <-> replica client + db->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) { + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { + // Database <-> replica might be behind, check and recover + client->TryCheckReplicaStateAsync(db->storage()); + } + }); + }); + }); +} + +} // namespace memgraph::dbms diff --git a/src/dbms/replication_client.hpp b/src/dbms/replication_client.hpp new file mode 100644 index 000000000..c1bac91a2 --- /dev/null +++ b/src/dbms/replication_client.hpp @@ -0,0 +1,21 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "dbms/dbms_handler.hpp" +#include "replication/replication_client.hpp" + +namespace memgraph::dbms { + +void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client); + +} // namespace memgraph::dbms diff --git a/src/dbms/replication_handler.cpp b/src/dbms/replication_handler.cpp index cff93fd6b..2cbe2c432 100644 --- a/src/dbms/replication_handler.cpp +++ b/src/dbms/replication_handler.cpp @@ -15,6 +15,7 @@ #include "dbms/dbms_handler.hpp" #include "dbms/inmemory/replication_handlers.hpp" #include "dbms/inmemory/storage_helper.hpp" +#include "dbms/replication_client.hpp" #include "replication/state.hpp" using memgraph::replication::ReplicationClientConfig; @@ -41,6 +42,8 @@ std::string RegisterReplicaErrorToString(RegisterReplicaError error) { } } // namespace +ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {} + bool ReplicationHandler::SetReplicationRoleMain() { auto const main_handler = [](RoleMainData const &) { // If we are already MAIN, we don't want to change anything @@ -56,42 +59,49 @@ bool ReplicationHandler::SetReplicationRoleMain() { // STEP 2) Change to MAIN // TODO: restore replication servers if false? - if (!repl_state_.SetReplicationRoleMain()) { + if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) { // TODO: Handle recovery on failure??? return false; } // STEP 3) We are now MAIN, update storage local epoch + const auto &epoch = + std::get(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_; dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); - storage->repl_storage_state_.epoch_ = std::get(std::as_const(repl_state_).ReplicationData()).epoch_; + storage->repl_storage_state_.epoch_ = epoch; }); return true; }; // TODO: under lock - return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData()); + return std::visit(utils::Overloaded{main_handler, replica_handler}, + dbms_handler_.ReplicationState().ReplicationData()); } bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) { // We don't want to restart the server if we're already a REPLICA - if (repl_state_.IsReplica()) { + if (dbms_handler_.ReplicationState().IsReplica()) { return false; } - // Remove registered replicas + // TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are + // deleting the replica. + // Remove database specific clients dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); }); }); + // Remove instance level clients + std::get(dbms_handler_.ReplicationState().ReplicationData()).registered_replicas_.clear(); // Creates the server - repl_state_.SetReplicationRoleReplica(config); + dbms_handler_.ReplicationState().SetReplicationRoleReplica(config); // Start const auto success = - std::visit(utils::Overloaded{[](auto) { + std::visit(utils::Overloaded{[](RoleMainData const &) { // ASSERT return false; }, @@ -104,36 +114,37 @@ bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication:: } return true; }}, - repl_state_.ReplicationData()); + dbms_handler_.ReplicationState().ReplicationData()); // TODO Handle error (restore to main?) return success; } auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config) -> memgraph::utils::BasicResult { - MG_ASSERT(repl_state_.IsMain(), "Only main instance can register a replica!"); + MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!"); - auto res = repl_state_.RegisterReplica(config); - switch (res) { - case memgraph::replication::RegisterReplicaError::NOT_MAIN: - MG_ASSERT(false, "Only main instance can register a replica!"); - return {}; - case memgraph::replication::RegisterReplicaError::NAME_EXISTS: - return memgraph::dbms::RegisterReplicaError::NAME_EXISTS; - case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS: - return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS; - case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED: - return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED; - case memgraph::replication::RegisterReplicaError::SUCCESS: - break; - } - - bool all_clients_good = true; + auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config); + if (instance_client.HasError()) switch (instance_client.GetError()) { + case memgraph::replication::RegisterReplicaError::NOT_MAIN: + MG_ASSERT(false, "Only main instance can register a replica!"); + return {}; + case memgraph::replication::RegisterReplicaError::NAME_EXISTS: + return memgraph::dbms::RegisterReplicaError::NAME_EXISTS; + case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS: + return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS; + case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED: + return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED; + case memgraph::replication::RegisterReplicaError::SUCCESS: + break; + } if (!allow_mt_repl && dbms_handler_.All().size() > 1) { spdlog::warn("Multi-tenant replication is currently not supported!"); } + bool all_clients_good = true; + + // Add database specific clients (NOTE Currently all databases are connected to each replica) dbms_handler_.ForEach([&](Database *db) { auto *storage = db->storage(); if (!allow_mt_repl && storage->id() != kDefaultDB) { @@ -143,18 +154,29 @@ auto ReplicationHandler::RegisterReplica(const memgraph::replication::Replicatio if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return; all_clients_good &= - storage->repl_storage_state_.replication_clients_.WithLock([storage, &config](auto &clients) -> bool { - auto client = storage->CreateReplicationClient(config, &storage->repl_storage_state_.epoch_); - client->Start(); - - if (client->State() == storage::replication::ReplicaState::INVALID) { + storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) { + auto client = std::make_unique(*instance_client.GetValue()); + client->Start(storage); + // After start the storage <-> replica state should be READY or RECOVERING (if correctly started) + // MAYBE_BEHIND isn't a statement of the current state, this is the default value + // Failed to start due to branching of MAIN and REPLICA + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { return false; } - clients.push_back(std::move(client)); + storage_clients.push_back(std::move(client)); return true; }); }); - if (!all_clients_good) return RegisterReplicaError::CONNECTION_FAILED; // TODO: this happen to 1 or many...what to do + + // NOTE Currently if any databases fails, we revert back + if (!all_clients_good) { + spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name); + UnregisterReplica(config.name); + return RegisterReplicaError::CONNECTION_FAILED; + } + + // No client error, start instance level client + StartReplicaClient(dbms_handler_, *instance_client.GetValue()); return {}; } @@ -163,60 +185,66 @@ auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterR return UnregisterReplicaResult::NOT_MAIN; }; auto const main_handler = [this, name](RoleMainData &mainData) -> UnregisterReplicaResult { - if (!repl_state_.TryPersistUnregisterReplica(name)) { + if (!dbms_handler_.ReplicationState().TryPersistUnregisterReplica(name)) { return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED; } - auto const n_unregistered = - std::erase_if(mainData.registered_replicas_, - [&](ReplicationClientConfig const ®istered_config) { return registered_config.name == name; }); - - dbms_handler_.ForEach([&](Database *db) { - db->storage()->repl_storage_state_.replication_clients_.WithLock( - [&](auto &clients) { std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); }); + // Remove database specific clients + dbms_handler_.ForEach([name](Database *db) { + db->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) { + std::erase_if(clients, [name](const auto &client) { return client->Name() == name; }); + }); }); - + // Remove instance level clients + auto const n_unregistered = + std::erase_if(mainData.registered_replicas_, [name](auto const &client) { return client.name_ == name; }); return n_unregistered != 0 ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER; }; - return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData()); + return std::visit(utils::Overloaded{main_handler, replica_handler}, + dbms_handler_.ReplicationState().ReplicationData()); } -auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { return repl_state_.GetRole(); } +auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { + return dbms_handler_.ReplicationState().GetRole(); +} -bool ReplicationHandler::IsMain() const { return repl_state_.IsMain(); } +bool ReplicationHandler::IsMain() const { return dbms_handler_.ReplicationState().IsMain(); } -bool ReplicationHandler::IsReplica() const { return repl_state_.IsReplica(); } +bool ReplicationHandler::IsReplica() const { return dbms_handler_.ReplicationState().IsReplica(); } -void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage) { +// Per storage +// NOTE Storage will connect to all replicas. Future work might change this +void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage) { spdlog::info("Restoring replication role."); /// MAIN - auto const recover_main = [&storage](RoleMainData const &mainData) { - for (const auto &config : mainData.registered_replicas_) { - spdlog::info("Replica {} restoration started for {}.", config.name, storage.id()); + auto const recover_main = [&storage](RoleMainData &mainData) { + // Each individual client has already been restored and started. Here we just go through each database and start its + // client + for (auto &instance_client : mainData.registered_replicas_) { + spdlog::info("Replica {} restoration started for {}.", instance_client.name_, storage.id()); - auto register_replica = [&storage](const memgraph::replication::ReplicationClientConfig &config) - -> memgraph::utils::BasicResult { - return storage.repl_storage_state_.replication_clients_.WithLock( - [&storage, &config](auto &clients) -> utils::BasicResult { - auto client = storage.CreateReplicationClient(config, &storage.repl_storage_state_.epoch_); - client->Start(); + const auto &ret = storage.repl_storage_state_.replication_clients_.WithLock( + [&](auto &storage_clients) -> utils::BasicResult { + auto client = std::make_unique(instance_client); + client->Start(&storage); + // After start the storage <-> replica state should be READY or RECOVERING (if correctly started) + // MAYBE_BEHIND isn't a statement of the current state, this is the default value + // Failed to start due to branching of MAIN and REPLICA + if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) { + spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", + instance_client.name_); + } + storage_clients.push_back(std::move(client)); + return {}; + }); - if (client->State() == storage::replication::ReplicaState::INVALID) { - spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", - client->Name()); - } - clients.push_back(std::move(client)); - return {}; - }); - }; - - auto ret = register_replica(config); if (ret.HasError()) { MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError()); - LOG_FATAL("Failure when restoring replica {}: {}.", config.name, RegisterReplicaErrorToString(ret.GetError())); + LOG_FATAL("Failure when restoring replica {}: {}.", instance_client.name_, + RegisterReplicaErrorToString(ret.GetError())); } - spdlog::info("Replica {} restored for {}.", config.name, storage.id()); + spdlog::info("Replica {} restored for {}.", instance_client.name_, storage.id()); } spdlog::info("Replication role restored to MAIN."); }; @@ -229,6 +257,6 @@ void RestoreReplication(const replication::ReplicationState &repl_state, storage recover_main, recover_replica, }, - std::as_const(repl_state).ReplicationData()); + repl_state.ReplicationData()); } } // namespace memgraph::dbms diff --git a/src/dbms/replication_handler.hpp b/src/dbms/replication_handler.hpp index e50c47969..dc95407b1 100644 --- a/src/dbms/replication_handler.hpp +++ b/src/dbms/replication_handler.hpp @@ -36,8 +36,7 @@ enum class UnregisterReplicaResult : uint8_t { /// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage /// TODO: extend to do multiple storages struct ReplicationHandler { - ReplicationHandler(memgraph::replication::ReplicationState &replState, DbmsHandler &dbms_handler) - : repl_state_(replState), dbms_handler_(dbms_handler) {} + explicit ReplicationHandler(DbmsHandler &dbms_handler); // as REPLICA, become MAIN bool SetReplicationRoleMain(); @@ -58,12 +57,11 @@ struct ReplicationHandler { bool IsReplica() const; private: - memgraph::replication::ReplicationState &repl_state_; DbmsHandler &dbms_handler_; }; /// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage /// TODO: extend to do multiple storages -void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage); +void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage); } // namespace memgraph::dbms diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 983dd61f9..ce43f634d 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -368,34 +368,17 @@ int main(int argc, char **argv) { std::unique_ptr auth_checker; auth_glue(&auth_, auth_handler, auth_checker); - memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(db_config)); - - memgraph::dbms::DbmsHandler dbms_handler(db_config, repl_state + memgraph::dbms::DbmsHandler dbms_handler(db_config #ifdef MG_ENTERPRISE , &auth_, FLAGS_data_recovery_on_startup, FLAGS_storage_delete_on_drop #endif ); auto db_acc = dbms_handler.Get(); - memgraph::query::InterpreterContext interpreter_context_(interp_config, &dbms_handler, &repl_state, - auth_handler.get(), auth_checker.get()); - MG_ASSERT(db_acc, "Failed to access the main database"); - // TODO: Move it somewhere better - // Startup replication state (if recovered at startup) - MG_ASSERT(std::visit(memgraph::utils::Overloaded{[](memgraph::replication::RoleMainData const &) { return true; }, - [&](memgraph::replication::RoleReplicaData const &data) { - // Register handlers - memgraph::dbms::InMemoryReplicationHandlers::Register( - &dbms_handler, *data.server); - if (!data.server->Start()) { - spdlog::error("Unable to start the replication server."); - return false; - } - return true; - }}, - repl_state.ReplicationData()), - "Replica recovery failure!"); + memgraph::query::InterpreterContext interpreter_context_( + interp_config, &dbms_handler, &dbms_handler.ReplicationState(), auth_handler.get(), auth_checker.get()); + MG_ASSERT(db_acc, "Failed to access the main database"); memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(), FLAGS_data_directory); diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index dc11c3887..d63736c85 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -3025,7 +3025,7 @@ class ReplicationQuery : public memgraph::query::Query { enum class SyncMode { SYNC, ASYNC }; - enum class ReplicaState { READY, REPLICATING, RECOVERY, INVALID }; + enum class ReplicaState { READY, REPLICATING, RECOVERY, MAYBE_BEHIND }; ReplicationQuery() = default; diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 354f13dc3..d957d4c2e 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -274,8 +274,7 @@ inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode class ReplQueryHandler final : public query::ReplicationQueryHandler { public: - explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler, memgraph::replication::ReplicationState *repl_state) - : dbms_handler_(dbms_handler), handler_{*repl_state, *dbms_handler} {} + explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler) : dbms_handler_(dbms_handler), handler_{*dbms_handler} {} /// @throw QueryRuntimeException if an error ocurred. void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { @@ -404,8 +403,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { case storage::replication::ReplicaState::RECOVERY: replica.state = ReplicationQuery::ReplicaState::RECOVERY; break; - case storage::replication::ReplicaState::INVALID: - replica.state = ReplicationQuery::ReplicaState::INVALID; + case storage::replication::ReplicaState::MAYBE_BEHIND: + replica.state = ReplicationQuery::ReplicaState::MAYBE_BEHIND; break; } @@ -713,8 +712,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters, dbms::DbmsHandler *dbms_handler, const query::InterpreterConfig &config, - std::vector *notifications, - memgraph::replication::ReplicationState *repl_state) { + std::vector *notifications) { // TODO: MemoryResource for EvaluationContext, it should probably be passed as // the argument to Callback. EvaluationContext evaluation_context; @@ -734,8 +732,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & notifications->emplace_back(SeverityLevel::WARNING, NotificationCode::REPLICA_PORT_WARNING, "Be careful the replication port must be different from the memgraph port!"); } - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, role = repl_query->role_, - maybe_port]() mutable { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, role = repl_query->role_, maybe_port]() mutable { handler.SetReplicationRole(role, maybe_port); return std::vector>(); }; @@ -747,7 +744,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: { callback.header = {"replication role"}; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}] { + callback.fn = [handler = ReplQueryHandler{dbms_handler}] { auto mode = handler.ShowReplicationRole(); switch (mode) { case ReplicationQuery::ReplicationRole::MAIN: { @@ -766,7 +763,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & auto socket_address = repl_query->socket_address_->Accept(evaluator); const auto replica_check_frequency = config.replication_replica_check_frequency; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, name, socket_address, sync_mode, + callback.fn = [handler = ReplQueryHandler{dbms_handler}, name, socket_address, sync_mode, replica_check_frequency]() mutable { handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency); return std::vector>(); @@ -777,7 +774,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } case ReplicationQuery::Action::DROP_REPLICA: { const auto &name = repl_query->replica_name_; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, name]() mutable { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, name]() mutable { handler.DropReplica(name); return std::vector>(); }; @@ -789,7 +786,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & callback.header = { "name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master", "state"}; - callback.fn = [handler = ReplQueryHandler{dbms_handler, repl_state}, replica_nfields = callback.header.size()] { + callback.fn = [handler = ReplQueryHandler{dbms_handler}, replica_nfields = callback.header.size()] { const auto &replicas = handler.ShowReplicas(); auto typed_replicas = std::vector>{}; typed_replicas.reserve(replicas.size()); @@ -822,7 +819,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & case ReplicationQuery::ReplicaState::RECOVERY: typed_replica.emplace_back("recovery"); break; - case ReplicationQuery::ReplicaState::INVALID: + case ReplicationQuery::ReplicaState::MAYBE_BEHIND: typed_replica.emplace_back("invalid"); break; } @@ -2263,15 +2260,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit_transaction, std::vector *notifications, dbms::DbmsHandler &dbms_handler, - const InterpreterConfig &config, - memgraph::replication::ReplicationState *repl_state) { + const InterpreterConfig &config) { if (in_explicit_transaction) { throw ReplicationModificationInMulticommandTxException(); } auto *replication_query = utils::Downcast(parsed_query.query); - auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, - notifications, repl_state); + auto callback = + HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, notifications); return PreparedQuery{callback.header, std::move(parsed_query.required_privileges), [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( @@ -3349,8 +3345,7 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB ¤t_db, InterpreterContext *interpreter_context, - std::optional> on_change_cb, - memgraph::replication::ReplicationState *repl_state) { + std::optional> on_change_cb) { #ifdef MG_ENTERPRISE if (!license::global_license_checker.IsEnterpriseValidFast()) { throw QueryException("Trying to use enterprise feature without a valid license."); @@ -3361,9 +3356,11 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur auto *query = utils::Downcast(parsed_query.query); auto *db_handler = interpreter_context->dbms_handler; + const bool is_replica = interpreter_context->repl_state->IsReplica(); + switch (query->action_) { case MultiDatabaseQuery::Action::CREATE: - if (repl_state->IsReplica()) { + if (is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{ @@ -3408,12 +3405,12 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur if (current_db.in_explicit_db_) { throw QueryException("Database switching is prohibited if session explicitly defines the used database"); } - if (!dbms::allow_mt_repl && repl_state->IsReplica()) { + if (!dbms::allow_mt_repl && is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{{"STATUS"}, std::move(parsed_query.required_privileges), - [db_name = query->db_name_, db_handler, ¤t_db, on_change_cb]( + [db_name = query->db_name_, db_handler, ¤t_db, on_change = std::move(on_change_cb)]( AnyStream *stream, std::optional n) -> std::optional { std::vector> status; std::string res; @@ -3423,7 +3420,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur res = "Already using " + db_name; } else { auto tmp = db_handler->Get(db_name); - if (on_change_cb) (*on_change_cb)(db_name); // Will trow if cb fails + if (on_change) (*on_change)(db_name); // Will trow if cb fails current_db.SetCurrentDB(std::move(tmp), false); res = "Using " + db_name; } @@ -3442,7 +3439,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur query->db_name_}; case MultiDatabaseQuery::Action::DROP: - if (repl_state->IsReplica()) { + if (is_replica) { throw QueryException("Query forbidden on the replica!"); } return PreparedQuery{ @@ -3765,9 +3762,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, &query_execution->notifications, current_db_); } else if (utils::Downcast(parsed_query.query)) { /// TODO: make replication DB agnostic - prepared_query = PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, - &query_execution->notifications, *interpreter_context_->dbms_handler, - interpreter_context_->config, interpreter_context_->repl_state); + prepared_query = + PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + *interpreter_context_->dbms_handler, interpreter_context_->config); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { @@ -3807,8 +3804,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, throw MultiDatabaseQueryInMulticommandTxException(); } /// SYSTEM (Replication) + INTERPRETER - prepared_query = PrepareMultiDatabaseQuery(std::move(parsed_query), current_db_, interpreter_context_, on_change_, - interpreter_context_->repl_state); + prepared_query = + PrepareMultiDatabaseQuery(std::move(parsed_query), current_db_, interpreter_context_, on_change_); } else if (utils::Downcast(parsed_query.query)) { /// SYSTEM PURE ("SHOW DATABASES") /// INTERPRETER (TODO: "SHOW DATABASE") diff --git a/src/replication/CMakeLists.txt b/src/replication/CMakeLists.txt index 772ae5591..597ed096a 100644 --- a/src/replication/CMakeLists.txt +++ b/src/replication/CMakeLists.txt @@ -6,8 +6,10 @@ target_sources(mg-replication include/replication/epoch.hpp include/replication/config.hpp include/replication/mode.hpp + include/replication/messages.hpp include/replication/role.hpp include/replication/status.hpp + include/replication/replication_client.hpp include/replication/replication_server.hpp PRIVATE @@ -15,6 +17,8 @@ target_sources(mg-replication epoch.cpp config.cpp status.cpp + messages.cpp + replication_client.cpp replication_server.cpp ) target_include_directories(mg-replication PUBLIC include) diff --git a/src/replication/include/replication/messages.hpp b/src/replication/include/replication/messages.hpp new file mode 100644 index 000000000..57cf29351 --- /dev/null +++ b/src/replication/include/replication/messages.hpp @@ -0,0 +1,44 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "rpc/messages.hpp" +#include "slk/serialization.hpp" + +namespace memgraph::replication { + +struct FrequentHeartbeatReq { + static const utils::TypeInfo kType; // TODO: make constexpr? + static const utils::TypeInfo &GetTypeInfo() { return kType; } // WHAT? + + static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); + static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); + FrequentHeartbeatReq() = default; +}; + +struct FrequentHeartbeatRes { + static const utils::TypeInfo kType; + static const utils::TypeInfo &GetTypeInfo() { return kType; } + + static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); + static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); + FrequentHeartbeatRes() = default; + explicit FrequentHeartbeatRes(bool success) : success(success) {} + + bool success; +}; + +using FrequentHeartbeatRpc = rpc::RequestResponse; + +void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder); + +} // namespace memgraph::replication diff --git a/src/replication/include/replication/replication_client.hpp b/src/replication/include/replication/replication_client.hpp new file mode 100644 index 000000000..16e1010bf --- /dev/null +++ b/src/replication/include/replication/replication_client.hpp @@ -0,0 +1,82 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "replication/config.hpp" +#include "replication/messages.hpp" +#include "rpc/client.hpp" +#include "utils/scheduler.hpp" +#include "utils/thread_pool.hpp" + +#include +#include + +namespace memgraph::replication { + +template +concept InvocableWithStringView = std::invocable; + +struct ReplicationClient { + explicit ReplicationClient(const memgraph::replication::ReplicationClientConfig &config); + + ~ReplicationClient(); + ReplicationClient(ReplicationClient const &) = delete; + ReplicationClient &operator=(ReplicationClient const &) = delete; + ReplicationClient(ReplicationClient &&) noexcept = delete; + ReplicationClient &operator=(ReplicationClient &&) noexcept = delete; + + template + void StartFrequentCheck(F &&callback) { + // Help the user to get the most accurate replica state possible. + if (replica_check_frequency_ > std::chrono::seconds(0)) { + replica_checker_.Run("Replica Checker", replica_check_frequency_, [this, cb = std::forward(callback)] { + try { + bool success = false; + { + auto stream{rpc_client_.Stream()}; + success = stream.AwaitResponse().success; + } + if (success) { + cb(name_); + } + } catch (const rpc::RpcFailedException &) { + // Nothing to do...wait for a reconnect + } + }); + } + } + + std::string name_; + communication::ClientContext rpc_context_; + rpc::Client rpc_client_; + std::chrono::seconds replica_check_frequency_; + + memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC}; + // This thread pool is used for background tasks so we don't + // block the main storage thread + // We use only 1 thread for 2 reasons: + // - background tasks ALWAYS contain some kind of RPC communication. + // We can't have multiple RPC communication from a same client + // because that's not logically valid (e.g. you cannot send a snapshot + // and WAL at a same time because WAL will arrive earlier and be applied + // before the snapshot which is not correct) + // - the implementation is simplified as we have a total control of what + // this pool is executing. Also, we can simply queue multiple tasks + // and be sure of the execution order. + // Not having mulitple possible threads in the same client allows us + // to ignore concurrency problems inside the client. + utils::ThreadPool thread_pool_{1}; + + utils::Scheduler replica_checker_; +}; + +} // namespace memgraph::replication diff --git a/src/replication/include/replication/replication_server.hpp b/src/replication/include/replication/replication_server.hpp index e9ca1b549..5ff41b8a5 100644 --- a/src/replication/include/replication/replication_server.hpp +++ b/src/replication/include/replication/replication_server.hpp @@ -17,30 +17,6 @@ namespace memgraph::replication { -struct FrequentHeartbeatReq { - static const utils::TypeInfo kType; // TODO: make constexpr? - static const utils::TypeInfo &GetTypeInfo() { return kType; } // WHAT? - - static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader); - static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder); - FrequentHeartbeatReq() = default; -}; - -struct FrequentHeartbeatRes { - static const utils::TypeInfo kType; - static const utils::TypeInfo &GetTypeInfo() { return kType; } - - static void Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader); - static void Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder); - FrequentHeartbeatRes() = default; - explicit FrequentHeartbeatRes(bool success) : success(success) {} - - bool success; -}; - -// TODO: move to own header -using FrequentHeartbeatRpc = rpc::RequestResponse; - class ReplicationServer { public: explicit ReplicationServer(const memgraph::replication::ReplicationServerConfig &config); diff --git a/src/replication/include/replication/state.hpp b/src/replication/include/replication/state.hpp index 0460d0a9d..76aec1053 100644 --- a/src/replication/include/replication/state.hpp +++ b/src/replication/include/replication/state.hpp @@ -11,19 +11,22 @@ #pragma once -#include -#include -#include -#include - #include "kvstore/kvstore.hpp" #include "replication/config.hpp" #include "replication/epoch.hpp" #include "replication/mode.hpp" +#include "replication/replication_client.hpp" #include "replication/role.hpp" #include "replication_server.hpp" #include "status.hpp" #include "utils/result.hpp" +#include "utils/synchronized.hpp" + +#include +#include +#include +#include +#include namespace memgraph::replication { @@ -32,8 +35,17 @@ enum class RolePersisted : uint8_t { UNKNOWN_OR_NO, YES }; enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, COULD_NOT_BE_PERSISTED, NOT_MAIN, SUCCESS }; struct RoleMainData { + RoleMainData() = default; + explicit RoleMainData(ReplicationEpoch e) : epoch_(std::move(e)) {} + ~RoleMainData() = default; + + RoleMainData(RoleMainData const &) = delete; + RoleMainData &operator=(RoleMainData const &) = delete; + RoleMainData(RoleMainData &&) = default; + RoleMainData &operator=(RoleMainData &&) = default; + ReplicationEpoch epoch_; - std::vector registered_replicas_; + std::list registered_replicas_{}; }; struct RoleReplicaData { @@ -41,8 +53,10 @@ struct RoleReplicaData { std::unique_ptr server; }; +// Global (instance) level object struct ReplicationState { explicit ReplicationState(std::optional durability_dir); + ~ReplicationState() = default; ReplicationState(ReplicationState const &) = delete; ReplicationState(ReplicationState &&) = delete; @@ -74,7 +88,7 @@ struct ReplicationState { // TODO: locked access auto ReplicationData() -> ReplicationData_t & { return replication_data_; } auto ReplicationData() const -> ReplicationData_t const & { return replication_data_; } - auto RegisterReplica(const ReplicationClientConfig &config) -> RegisterReplicaError; + utils::BasicResult RegisterReplica(const ReplicationClientConfig &config); bool SetReplicationRoleMain(); diff --git a/src/replication/messages.cpp b/src/replication/messages.cpp new file mode 100644 index 000000000..4503e9df2 --- /dev/null +++ b/src/replication/messages.cpp @@ -0,0 +1,65 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/messages.hpp" +#include "rpc/messages.hpp" +#include "slk/serialization.hpp" +#include "slk/streams.hpp" + +namespace memgraph::slk { +// Serialize code for FrequentHeartbeatRes +void Save(const memgraph::replication::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self.success, builder); +} +void Load(memgraph::replication::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(&self->success, reader); +} + +// Serialize code for FrequentHeartbeatReq +void Save(const memgraph::replication::FrequentHeartbeatReq & /*self*/, memgraph::slk::Builder * /*builder*/) { + /* Nothing to serialize */ +} +void Load(memgraph::replication::FrequentHeartbeatReq * /*self*/, memgraph::slk::Reader * /*reader*/) { + /* Nothing to serialize */ +} + +} // namespace memgraph::slk + +namespace memgraph::replication { + +constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_REQ, "FrequentHeartbeatReq", + nullptr}; + +constexpr utils::TypeInfo FrequentHeartbeatRes::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_RES, "FrequentHeartbeatRes", + nullptr}; + +void FrequentHeartbeatReq::Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self, builder); +} +void FrequentHeartbeatReq::Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(self, reader); +} +void FrequentHeartbeatRes::Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { + memgraph::slk::Save(self, builder); +} +void FrequentHeartbeatRes::Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { + memgraph::slk::Load(self, reader); +} + +void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { + FrequentHeartbeatReq req; + FrequentHeartbeatReq::Load(&req, req_reader); + memgraph::slk::Load(&req, req_reader); + FrequentHeartbeatRes res{true}; + memgraph::slk::Save(res, res_builder); +} + +} // namespace memgraph::replication diff --git a/src/replication/replication_client.cpp b/src/replication/replication_client.cpp new file mode 100644 index 000000000..d14250c2a --- /dev/null +++ b/src/replication/replication_client.cpp @@ -0,0 +1,40 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/replication_client.hpp" + +namespace memgraph::replication { + +static auto CreateClientContext(const memgraph::replication::ReplicationClientConfig &config) + -> communication::ClientContext { + return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file} + : communication::ClientContext{}; +} + +ReplicationClient::ReplicationClient(const memgraph::replication::ReplicationClientConfig &config) + : name_{config.name}, + rpc_context_{CreateClientContext(config)}, + rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port), + &rpc_context_}, + replica_check_frequency_{config.replica_check_frequency}, + mode_{config.mode} {} + +ReplicationClient::~ReplicationClient() { + auto endpoint = rpc_client_.Endpoint(); + try { + spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port); + } catch (...) { + // Logging can throw. Not a big deal, just ignore. + } + thread_pool_.Shutdown(); +} + +} // namespace memgraph::replication diff --git a/src/replication/replication_server.cpp b/src/replication/replication_server.cpp index 7d0ff3cc2..f79ea2add 100644 --- a/src/replication/replication_server.cpp +++ b/src/replication/replication_server.cpp @@ -10,25 +10,7 @@ // licenses/APL.txt. #include "replication/replication_server.hpp" -#include "rpc/messages.hpp" -#include "slk/serialization.hpp" -#include "slk/streams.hpp" - -namespace memgraph::slk { - -// Serialize code for FrequentHeartbeatRes -void Save(const memgraph::replication::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self.success, builder); -} -void Load(memgraph::replication::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(&self->success, reader); -} - -// Serialize code for FrequentHeartbeatReq -void Save(const memgraph::replication::FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) {} -void Load(memgraph::replication::FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) {} - -} // namespace memgraph::slk +#include "replication/messages.hpp" namespace memgraph::replication { namespace { @@ -39,13 +21,6 @@ auto CreateServerContext(const memgraph::replication::ReplicationServerConfig &c : communication::ServerContext{}; } -void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { - FrequentHeartbeatReq req; - memgraph::slk::Load(&req, req_reader); - FrequentHeartbeatRes res{true}; - memgraph::slk::Save(res, res_builder); -} - // NOTE: The replication server must have a single thread for processing // because there is no need for more processing threads - each replica can // have only a single main server. Also, the single-threaded guarantee @@ -53,25 +28,6 @@ void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder constexpr auto kReplicationServerThreads = 1; } // namespace -constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_REQ, "FrequentHeartbeatReq", - nullptr}; - -constexpr utils::TypeInfo FrequentHeartbeatRes::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_RES, "FrequentHeartbeatRes", - nullptr}; - -void FrequentHeartbeatReq::Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self, builder); -} -void FrequentHeartbeatReq::Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(self, reader); -} -void FrequentHeartbeatRes::Save(const FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) { - memgraph::slk::Save(self, builder); -} -void FrequentHeartbeatRes::Load(FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) { - memgraph::slk::Load(self, reader); -} - ReplicationServer::ReplicationServer(const memgraph::replication::ReplicationServerConfig &config) : rpc_server_context_{CreateServerContext(config)}, rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_, diff --git a/src/replication/state.cpp b/src/replication/state.cpp index 4551eba7e..60c390e17 100644 --- a/src/replication/state.cpp +++ b/src/replication/state.cpp @@ -11,9 +11,11 @@ #include "replication/state.hpp" +#include "replication/replication_client.hpp" #include "replication/replication_server.hpp" #include "replication/status.hpp" #include "utils/file.hpp" +#include "utils/result.hpp" #include "utils/variant_helpers.hpp" constexpr auto kReplicationDirectory = std::string_view{"replication"}; @@ -125,12 +127,9 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t { return std::visit( utils::Overloaded{ [&](durability::MainRole &&r) -> FetchReplicationResult_t { - auto res = RoleMainData{ - .epoch_ = std::move(r.epoch), - }; + auto res = RoleMainData{std::move(r.epoch)}; auto b = durability_->begin(durability::kReplicationReplicaPrefix); auto e = durability_->end(durability::kReplicationReplicaPrefix); - res.registered_replicas_.reserve(durability_->Size(durability::kReplicationReplicaPrefix)); for (; b != e; ++b) { auto const &[replica_name, replica_data] = *b; auto json = nlohmann::json::parse(replica_data, nullptr, false); @@ -141,7 +140,8 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t { if (key_name != data.config.name) { return FetchReplicationError::PARSE_ERROR; } - res.registered_replicas_.emplace_back(std::move(data.config)); + // Instance clients + res.registered_replicas_.emplace_back(data.config); } catch (...) { return FetchReplicationError::PARSE_ERROR; } @@ -221,7 +221,7 @@ bool ReplicationState::SetReplicationRoleMain() { if (!TryPersistRoleMain(new_epoch)) { return false; } - replication_data_ = RoleMainData{.epoch_ = ReplicationEpoch{new_epoch}}; + replication_data_ = RoleMainData{ReplicationEpoch{new_epoch}}; return true; } @@ -233,16 +233,14 @@ bool ReplicationState::SetReplicationRoleReplica(const ReplicationServerConfig & return true; } -auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> RegisterReplicaError { - auto const replica_handler = [](RoleReplicaData const &) -> RegisterReplicaError { - return RegisterReplicaError::NOT_MAIN; - }; - auto const main_handler = [this, &config](RoleMainData &mainData) -> RegisterReplicaError { +utils::BasicResult ReplicationState::RegisterReplica( + const ReplicationClientConfig &config) { + auto const replica_handler = [](RoleReplicaData const &) { return RegisterReplicaError::NOT_MAIN; }; + ReplicationClient *client{nullptr}; + auto const main_handler = [&client, &config, this](RoleMainData &mainData) -> RegisterReplicaError { // name check auto name_check = [&config](auto const &replicas) { - auto name_matches = [&name = config.name](ReplicationClientConfig const ®istered_config) { - return registered_config.name == name; - }; + auto name_matches = [&name = config.name](auto const &replica) { return replica.name_ == name; }; return std::any_of(replicas.begin(), replicas.end(), name_matches); }; if (name_check(mainData.registered_replicas_)) { @@ -251,8 +249,9 @@ auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> // endpoint check auto endpoint_check = [&](auto const &replicas) { - auto endpoint_matches = [&config](ReplicationClientConfig const ®istered_config) { - return registered_config.ip_address == config.ip_address && registered_config.port == config.port; + auto endpoint_matches = [&config](auto const &replica) { + const auto &ep = replica.rpc_client_.Endpoint(); + return ep.address == config.ip_address && ep.port == config.port; }; return std::any_of(replicas.begin(), replicas.end(), endpoint_matches); }; @@ -266,10 +265,14 @@ auto ReplicationState::RegisterReplica(const ReplicationClientConfig &config) -> } // set - mainData.registered_replicas_.emplace_back(config); + client = &mainData.registered_replicas_.emplace_back(config); return RegisterReplicaError::SUCCESS; }; - return std::visit(utils::Overloaded{main_handler, replica_handler}, replication_data_); + const auto &res = std::visit(utils::Overloaded{main_handler, replica_handler}, replication_data_); + if (res == RegisterReplicaError::SUCCESS) { + return client; + } + return res; } } // namespace memgraph::replication diff --git a/src/rpc/client.hpp b/src/rpc/client.hpp index f727391ac..1fd3fff8d 100644 --- a/src/rpc/client.hpp +++ b/src/rpc/client.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "communication/client.hpp" #include "io/network/endpoint.hpp" @@ -41,16 +42,25 @@ class Client { StreamHandler(Client *self, std::unique_lock &&guard, std::function res_load) - : self_(self), - guard_(std::move(guard)), - req_builder_([self](const uint8_t *data, size_t size, bool have_more) { - if (!self->client_->Write(data, size, have_more)) throw GenericRpcFailedException(); - }), - res_load_(res_load) {} + : self_(self), guard_(std::move(guard)), req_builder_(GenBuilderCallback(self, this)), res_load_(res_load) {} public: - StreamHandler(StreamHandler &&) noexcept = default; - StreamHandler &operator=(StreamHandler &&) noexcept = default; + StreamHandler(StreamHandler &&other) noexcept + : self_{std::exchange(other.self_, nullptr)}, + defunct_{std::exchange(other.defunct_, true)}, + guard_{std::move(other.guard_)}, + req_builder_{std::move(other.req_builder_), GenBuilderCallback(self_, this)}, + res_load_{std::move(other.res_load_)} {} + StreamHandler &operator=(StreamHandler &&other) noexcept { + if (&other != this) { + self_ = std::exchange(other.self_, nullptr); + defunct_ = std::exchange(other.defunct_, true); + guard_ = std::move(other.guard_); + req_builder_ = slk::Builder(std::move(other.req_builder_, GenBuilderCallback(self_, this))); + res_load_ = std::move(other.res_load_); + } + return *this; + } StreamHandler(const StreamHandler &) = delete; StreamHandler &operator=(const StreamHandler &) = delete; @@ -70,10 +80,18 @@ class Client { while (true) { auto ret = slk::CheckStreamComplete(self_->client_->GetData(), self_->client_->GetDataSize()); if (ret.status == slk::StreamStatus::INVALID) { + // Logically invalid state, connection is still up, defunct stream and release + defunct_ = true; + guard_.unlock(); throw GenericRpcFailedException(); - } else if (ret.status == slk::StreamStatus::PARTIAL) { + } + if (ret.status == slk::StreamStatus::PARTIAL) { if (!self_->client_->Read(ret.stream_size - self_->client_->GetDataSize(), /* exactly_len = */ false)) { + // Failed connection, abort and let somebody retry in the future + defunct_ = true; + self_->Abort(); + guard_.unlock(); throw GenericRpcFailedException(); } } else { @@ -103,7 +121,9 @@ class Client { // Check the response ID. if (res_id != res_type.id && res_id != utils::TypeId::UNKNOWN) { spdlog::error("Message response was of unexpected type"); - self_->client_ = std::nullopt; + // Logically invalid state, connection is still up, defunct stream and release + defunct_ = true; + guard_.unlock(); throw GenericRpcFailedException(); } @@ -112,8 +132,23 @@ class Client { return res_load_(&res_reader); } + bool IsDefunct() const { return defunct_; } + private: + static auto GenBuilderCallback(Client *client, StreamHandler *self) { + return [client, self](const uint8_t *data, size_t size, bool have_more) { + if (self->defunct_) throw GenericRpcFailedException(); + if (!client->client_->Write(data, size, have_more)) { + self->defunct_ = true; + client->Abort(); + self->guard_.unlock(); + throw GenericRpcFailedException(); + } + }; + } + Client *self_; + bool defunct_ = false; std::unique_lock guard_; slk::Builder req_builder_; std::function res_load_; @@ -179,7 +214,7 @@ class Client { TRequestResponse::Request::Save(request, handler.GetBuilder()); // Return the handler to the user. - return std::move(handler); + return handler; } /// Call a previously defined and registered RPC call. This function can diff --git a/src/slk/streams.cpp b/src/slk/streams.cpp index 5125d635a..dc5ef8c3c 100644 --- a/src/slk/streams.cpp +++ b/src/slk/streams.cpp @@ -30,7 +30,7 @@ void Builder::Save(const uint8_t *data, uint64_t size) { to_write = kSegmentMaxDataSize - pos_; } - memcpy(segment_ + sizeof(SegmentSize) + pos_, data + offset, to_write); + memcpy(segment_.data() + sizeof(SegmentSize) + pos_, data + offset, to_write); size -= to_write; pos_ += to_write; @@ -48,15 +48,15 @@ void Builder::FlushSegment(bool final_segment) { size_t total_size = sizeof(SegmentSize) + pos_; SegmentSize size = pos_; - memcpy(segment_, &size, sizeof(SegmentSize)); + memcpy(segment_.data(), &size, sizeof(SegmentSize)); if (final_segment) { SegmentSize footer = 0; - memcpy(segment_ + total_size, &footer, sizeof(SegmentSize)); + memcpy(segment_.data() + total_size, &footer, sizeof(SegmentSize)); total_size += sizeof(SegmentSize); } - write_func_(segment_, total_size, !final_segment); + write_func_(segment_.data(), total_size, !final_segment); pos_ = 0; } diff --git a/src/slk/streams.hpp b/src/slk/streams.hpp index 587b7830b..691189443 100644 --- a/src/slk/streams.hpp +++ b/src/slk/streams.hpp @@ -46,7 +46,11 @@ static_assert(kSegmentMaxDataSize <= std::numeric_limits::max(), /// Builder used to create a SLK segment stream. class Builder { public: - Builder(std::function write_func); + explicit Builder(std::function write_func); + Builder(Builder &&other, std::function write_func) + : write_func_{std::move(write_func)}, pos_{std::exchange(other.pos_, 0)}, segment_{other.segment_} { + other.write_func_ = [](const uint8_t *, size_t, bool) { /* Moved builder is defunct, no write possible */ }; + } /// Function used internally by SLK to serialize the data. void Save(const uint8_t *data, uint64_t size); @@ -59,7 +63,7 @@ class Builder { std::function write_func_; size_t pos_{0}; - uint8_t segment_[kSegmentMaxTotalSize]; + std::array segment_; }; /// Exception that will be thrown if segments can't be decoded from the byte diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 9f6d8d4d7..150a02cc7 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -39,6 +39,7 @@ add_library(mg-storage-v2 STATIC replication/slk.cpp replication/rpc.cpp replication/replication_storage_state.cpp - inmemory/replication/replication_client.cpp + inmemory/replication/recovery.cpp ) + target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory) diff --git a/src/storage/v2/disk/storage.hpp b/src/storage/v2/disk/storage.hpp index 3575e685d..e93566c09 100644 --- a/src/storage/v2/disk/storage.hpp +++ b/src/storage/v2/disk/storage.hpp @@ -313,12 +313,6 @@ class DiskStorage final : public Storage { uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); - auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig & /*config*/, - const memgraph::replication::ReplicationEpoch * /*current_epoch*/) - -> std::unique_ptr override { - throw utils::BasicException("Disk storage mode does not support replication."); - } - std::unique_ptr kvstore_; DurableMetadata durable_metadata_; EdgeImportMode edge_import_status_{EdgeImportMode::INACTIVE}; diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index 1240bd52e..9a43a2876 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -96,6 +96,7 @@ std::vector GetSnapshotFiles(const std::filesystem::path MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); } + std::sort(snapshot_files.begin(), snapshot_files.end()); return snapshot_files; } @@ -106,13 +107,17 @@ std::optional> GetWalFiles(const std::filesystem: std::vector wal_files; std::error_code error_code; + // There could be multiple "current" WAL files, the "_current" tag just means that the previous session didn't + // finalize. We cannot skip based on name, will be able to skip based on invalid data or sequence number, so the + // actual current wal will be skipped for (const auto &item : std::filesystem::directory_iterator(wal_directory, error_code)) { if (!item.is_regular_file()) continue; try { auto info = ReadWalInfo(item.path()); - if ((uuid.empty() || info.uuid == uuid) && (!current_seq_num || info.seq_num < *current_seq_num)) + if ((uuid.empty() || info.uuid == uuid) && (!current_seq_num || info.seq_num < *current_seq_num)) { wal_files.emplace_back(info.seq_num, info.from_timestamp, info.to_timestamp, std::move(info.uuid), std::move(info.epoch_id), item.path()); + } } catch (const RecoveryFailure &e) { spdlog::warn("Failed to read {}", item.path()); continue; @@ -120,6 +125,7 @@ std::optional> GetWalFiles(const std::filesystem: } MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message()); + // Sort based on the sequence number, not the file name std::sort(wal_files.begin(), wal_files.end()); return std::move(wal_files); } @@ -246,8 +252,6 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di std::optional snapshot_timestamp; if (!snapshot_files.empty()) { spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory); - // Order the files by name - std::sort(snapshot_files.begin(), snapshot_files.end()); // UUID used for durability is the UUID of the last snapshot file. *uuid = snapshot_files.back().uuid; diff --git a/src/storage/v2/inmemory/replication/recovery.cpp b/src/storage/v2/inmemory/replication/recovery.cpp new file mode 100644 index 000000000..b62fdc2f9 --- /dev/null +++ b/src/storage/v2/inmemory/replication/recovery.cpp @@ -0,0 +1,238 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/inmemory/replication/recovery.hpp" +#include +#include +#include +#include +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/replication/recovery.hpp" +#include "utils/on_scope_exit.hpp" +#include "utils/variant_helpers.hpp" + +namespace memgraph::storage { + +// Handler for transferring the current WAL file whose data is +// contained in the internal buffer and the file. +class InMemoryCurrentWalHandler { + public: + explicit InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client); + void AppendFilename(const std::string &filename); + + void AppendSize(size_t size); + + void AppendFileData(utils::InputFile *file); + + void AppendBufferData(const uint8_t *buffer, size_t buffer_size); + + /// @throw rpc::RpcFailedException + replication::CurrentWalRes Finalize(); + + private: + rpc::Client::StreamHandler stream_; +}; + +////// CurrentWalHandler ////// +InMemoryCurrentWalHandler::InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client) + : stream_(rpc_client.Stream(storage->id())) {} + +void InMemoryCurrentWalHandler::AppendFilename(const std::string &filename) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteString(filename); +} + +void InMemoryCurrentWalHandler::AppendSize(const size_t size) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteUint(size); +} + +void InMemoryCurrentWalHandler::AppendFileData(utils::InputFile *file) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteFileData(file); +} + +void InMemoryCurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) { + replication::Encoder encoder(stream_.GetBuilder()); + encoder.WriteBuffer(buffer, buffer_size); +} + +replication::CurrentWalRes InMemoryCurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } + +////// ReplicationClient Helpers ////// +replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, + const std::vector &wal_files) { + MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); + auto stream = client.Stream(std::move(db_name), wal_files.size()); + replication::Encoder encoder(stream.GetBuilder()); + for (const auto &wal : wal_files) { + spdlog::debug("Sending wal file: {}", wal); + encoder.WriteFile(wal); + } + return stream.AwaitResponse(); +} + +replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path) { + auto stream = client.Stream(std::move(db_name)); + replication::Encoder encoder(stream.GetBuilder()); + encoder.WriteFile(path); + return stream.AwaitResponse(); +} + +uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file) { + InMemoryCurrentWalHandler stream{storage, client}; + stream.AppendFilename(wal_file.Path().filename()); + utils::InputFile file; + MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file at {}!", wal_file.Path()); + const auto [buffer, buffer_size] = wal_file.CurrentFileBuffer(); + stream.AppendSize(file.GetSize() + buffer_size); + stream.AppendFileData(&file); + stream.AppendBufferData(buffer, buffer_size); + auto response = stream.Finalize(); + return response.current_commit_timestamp; +} + +/// This method tries to find the optimal path for recoverying a single replica. +/// Based on the last commit transfered to replica it tries to update the +/// replica using durability files - WALs and Snapshots. WAL files are much +/// smaller in size as they contain only the Deltas (changes) made during the +/// transactions while Snapshots contain all the data. For that reason we prefer +/// WALs as much as possible. As the WAL file that is currently being updated +/// can change during the process we ignore it as much as possible. Also, it +/// uses the transaction lock so locking it can be really expensive. After we +/// fetch the list of finalized WALs, we try to find the longest chain of +/// sequential WALs, starting from the latest one, that will update the recovery +/// with the all missed updates. If the WAL chain cannot be created, replica is +/// behind by a lot, so we use the regular recovery process, we send the latest +/// snapshot and all the necessary WAL files, starting from the newest WAL that +/// contains a timestamp before the snapshot. If we registered the existence of +/// the current WAL, we add the sequence number we read from it to the recovery +/// process. After all the other steps are finished, if the current WAL contains +/// the same sequence number, it's the same WAL we read while fetching the +/// recovery steps, so we can safely send it to the replica. +/// We assume that the property of preserving at least 1 WAL before the snapshot +/// is satisfied as we extract the timestamp information from it. +std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage) { + std::vector recovery_steps; + auto locker_acc = file_locker->Access(); + + // First check if we can recover using the current wal file only + // otherwise save the seq_num of the current wal file + // This lock is also necessary to force the missed transaction to finish. + std::optional current_wal_seq_num; + std::optional current_wal_from_timestamp; + + std::unique_lock transaction_guard( + storage->engine_lock_); // Hold the storage lock so the current wal file cannot be changed + (void)locker_acc.AddPath(storage->wal_directory_); // Protect all WALs from being deleted + + if (storage->wal_file_) { + current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); + current_wal_from_timestamp.emplace(storage->wal_file_->FromTimestamp()); + // No need to hold the lock since the current WAL is present and we can simply skip them + transaction_guard.unlock(); + } + + // Read in finalized WAL files (excluding the current/active WAL) + utils::OnScopeExit + release_wal_dir( // Each individually used file will be locked, so at the end, the dir can be released + [&locker_acc, &wal_dir = storage->wal_directory_]() { (void)locker_acc.RemovePath(wal_dir); }); + // Get WAL files, ordered by timestamp, from oldest to newest + auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); + MG_ASSERT(wal_files, "Wal files could not be loaded"); + if (transaction_guard.owns_lock()) + transaction_guard.unlock(); // In case we didn't have a current wal file, we can unlock only now since there is no + // guarantee what we'll see after we add the wal file + + // Read in snapshot files + (void)locker_acc.AddPath(storage->snapshot_directory_); // Protect all snapshots from being deleted + utils::OnScopeExit + release_snapshot_dir( // Each individually used file will be locked, so at the end, the dir can be released + [&locker_acc, &snapshot_dir = storage->snapshot_directory_]() { (void)locker_acc.RemovePath(snapshot_dir); }); + auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); + std::optional latest_snapshot{}; + if (!snapshot_files.empty()) { + latest_snapshot.emplace(std::move(snapshot_files.back())); + } + + auto add_snapshot = [&]() { + if (!latest_snapshot) return; + const auto lock_success = locker_acc.AddPath(latest_snapshot->path); + MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant snapshot path."); + recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); + }; + + // Check if we need the snapshot or if the WAL chain is enough + if (!wal_files->empty()) { + // Find WAL chain that contains the replica's commit timestamp + auto wal_chain_it = wal_files->rbegin(); + auto prev_seq{wal_chain_it->seq_num}; + for (; wal_chain_it != wal_files->rend(); ++wal_chain_it) { + if (prev_seq - wal_chain_it->seq_num > 1) { + // Broken chain, must have a snapshot that covers the missing commits + if (wal_chain_it->from_timestamp > replica_commit) { + // Chain does not go far enough, check the snapshot + MG_ASSERT(latest_snapshot, "Missing snapshot, while the WAL chain does not cover enough time."); + // Check for a WAL file that connects the snapshot to the chain + for (;; --wal_chain_it) { + // Going from the newest WAL files, find the first one that has a from_timestamp older than the snapshot + // NOTE: It could be that the only WAL needed is the current one + if (wal_chain_it->from_timestamp <= latest_snapshot->start_timestamp) { + break; + } + if (wal_chain_it == wal_files->rbegin()) break; + } + // Add snapshot to recovery steps + add_snapshot(); + } + break; + } + + if (wal_chain_it->to_timestamp <= replica_commit) { + // Got to a WAL that is older than what we need to recover the replica + break; + } + + prev_seq = wal_chain_it->seq_num; + } + + // Copy and lock the chain part we need, from oldest to newest + RecoveryWals rw{}; + rw.reserve(std::distance(wal_files->rbegin(), wal_chain_it)); + for (auto wal_it = wal_chain_it.base(); wal_it != wal_files->end(); ++wal_it) { + const auto lock_success = locker_acc.AddPath(wal_it->path); + MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant WAL path."); + rw.emplace_back(std::move(wal_it->path)); + } + if (!rw.empty()) { + recovery_steps.emplace_back(std::in_place_type_t{}, std::move(rw)); + } + + } else { + // No WAL chain, check if we need the snapshot + if (!current_wal_from_timestamp || replica_commit < *current_wal_from_timestamp) { + // No current wal or current wal too new + add_snapshot(); + } + } + + // In all cases, if we have a current wal file we need to use itW + if (current_wal_seq_num) { + // NOTE: File not handled directly, so no need to lock it + recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); + } + + return recovery_steps; +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/recovery.hpp b/src/storage/v2/inmemory/replication/recovery.hpp new file mode 100644 index 000000000..2025800ab --- /dev/null +++ b/src/storage/v2/inmemory/replication/recovery.hpp @@ -0,0 +1,32 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/replication/recovery.hpp" +#include "storage/v2/replication/replication_client.hpp" + +namespace memgraph::storage { +class InMemoryStorage; + +////// ReplicationClient Helpers ////// + +replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, + const std::vector &wal_files); + +replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path); + +uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file); + +auto GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage) -> std::vector; + +} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.cpp b/src/storage/v2/inmemory/replication/replication_client.cpp deleted file mode 100644 index b8ecc1c72..000000000 --- a/src/storage/v2/inmemory/replication/replication_client.cpp +++ /dev/null @@ -1,349 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v2/inmemory/replication/replication_client.hpp" - -#include "storage/v2/durability/durability.hpp" -#include "storage/v2/inmemory/storage.hpp" - -namespace memgraph::storage { - -namespace { -template -[[maybe_unused]] inline constexpr bool always_false_v = false; -} // namespace - -// Handler for transfering the current WAL file whose data is -// contained in the internal buffer and the file. -class CurrentWalHandler { - public: - explicit CurrentWalHandler(ReplicationClient *self); - void AppendFilename(const std::string &filename); - - void AppendSize(size_t size); - - void AppendFileData(utils::InputFile *file); - - void AppendBufferData(const uint8_t *buffer, size_t buffer_size); - - /// @throw rpc::RpcFailedException - replication::CurrentWalRes Finalize(); - - private: - ReplicationClient *self_; - rpc::Client::StreamHandler stream_; -}; - -////// CurrentWalHandler ////// -CurrentWalHandler::CurrentWalHandler(ReplicationClient *self) - : self_(self), stream_(self_->rpc_client_.Stream(self->GetStorageId())) {} - -void CurrentWalHandler::AppendFilename(const std::string &filename) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteString(filename); -} - -void CurrentWalHandler::AppendSize(const size_t size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteUint(size); -} - -void CurrentWalHandler::AppendFileData(utils::InputFile *file) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteFileData(file); -} - -void CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) { - replication::Encoder encoder(stream_.GetBuilder()); - encoder.WriteBuffer(buffer, buffer_size); -} - -replication::CurrentWalRes CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } - -////// ReplicationClient Helpers ////// - -replication::WalFilesRes TransferWalFiles(std::string db_name, rpc::Client &client, - const std::vector &wal_files) { - MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); - auto stream = client.Stream(std::move(db_name), wal_files.size()); - replication::Encoder encoder(stream.GetBuilder()); - for (const auto &wal : wal_files) { - spdlog::debug("Sending wal file: {}", wal); - encoder.WriteFile(wal); - } - return stream.AwaitResponse(); -} - -replication::SnapshotRes TransferSnapshot(std::string db_name, rpc::Client &client, const std::filesystem::path &path) { - auto stream = client.Stream(std::move(db_name)); - replication::Encoder encoder(stream.GetBuilder()); - encoder.WriteFile(path); - return stream.AwaitResponse(); -} - -uint64_t ReplicateCurrentWal(CurrentWalHandler &stream, durability::WalFile const &wal_file) { - stream.AppendFilename(wal_file.Path().filename()); - utils::InputFile file; - MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file!"); - const auto [buffer, buffer_size] = wal_file.CurrentFileBuffer(); - stream.AppendSize(file.GetSize() + buffer_size); - stream.AppendFileData(&file); - stream.AppendBufferData(buffer, buffer_size); - auto response = stream.Finalize(); - return response.current_commit_timestamp; -} - -////// ReplicationClient ////// - -InMemoryReplicationClient::InMemoryReplicationClient(InMemoryStorage *storage, - const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch) - : ReplicationClient{storage, config, epoch} {} - -void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { - spdlog::debug("Starting replica recover"); - auto *storage = static_cast(storage_); - while (true) { - auto file_locker = storage->file_retainer_.AddLocker(); - - const auto steps = GetRecoverySteps(replica_commit, &file_locker); - int i = 0; - for (const InMemoryReplicationClient::RecoveryStep &recovery_step : steps) { - spdlog::trace("Recovering in step: {}", i++); - try { - std::visit( - [&, this](T &&arg) { - using StepType = std::remove_cvref_t; - if constexpr (std::is_same_v) { // TODO: split into 3 overloads - spdlog::debug("Sending the latest snapshot file: {}", arg); - auto response = TransferSnapshot(storage->id(), rpc_client_, arg); - replica_commit = response.current_commit_timestamp; - } else if constexpr (std::is_same_v) { - spdlog::debug("Sending the latest wal files"); - auto response = TransferWalFiles(storage->id(), rpc_client_, arg); - replica_commit = response.current_commit_timestamp; - spdlog::debug("Wal files successfully transferred."); - } else if constexpr (std::is_same_v) { - std::unique_lock transaction_guard(storage->engine_lock_); - if (storage->wal_file_ && storage->wal_file_->SequenceNumber() == arg.current_wal_seq_num) { - storage->wal_file_->DisableFlushing(); - transaction_guard.unlock(); - spdlog::debug("Sending current wal file"); - auto streamHandler = CurrentWalHandler{this}; - replica_commit = ReplicateCurrentWal(streamHandler, *storage->wal_file_); - storage->wal_file_->EnableFlushing(); - } else { - spdlog::debug("Cannot recover using current wal file"); - } - } else { - static_assert(always_false_v, "Missing type from variant visitor"); - } - }, - recovery_step); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - return; - } - } - - spdlog::trace("Current timestamp on replica: {}", replica_commit); - // To avoid the situation where we read a correct commit timestamp in - // one thread, and after that another thread commits a different a - // transaction and THEN we set the state to READY in the first thread, - // we set this lock before checking the timestamp. - // We will detect that the state is invalid during the next commit, - // because replication::AppendDeltasRpc sends the last commit timestamp which - // replica checks if it's the same last commit timestamp it received - // and we will go to recovery. - // By adding this lock, we can avoid that, and go to RECOVERY immediately. - std::unique_lock client_guard{client_lock_}; - const auto last_commit_timestamp = LastCommitTimestamp(); - SPDLOG_INFO("Replica timestamp: {}", replica_commit); - SPDLOG_INFO("Last commit: {}", last_commit_timestamp); - if (last_commit_timestamp == replica_commit) { - replica_state_.store(replication::ReplicaState::READY); - return; - } - } -} - -/// This method tries to find the optimal path for recoverying a single replica. -/// Based on the last commit transfered to replica it tries to update the -/// replica using durability files - WALs and Snapshots. WAL files are much -/// smaller in size as they contain only the Deltas (changes) made during the -/// transactions while Snapshots contain all the data. For that reason we prefer -/// WALs as much as possible. As the WAL file that is currently being updated -/// can change during the process we ignore it as much as possible. Also, it -/// uses the transaction lock so locking it can be really expensive. After we -/// fetch the list of finalized WALs, we try to find the longest chain of -/// sequential WALs, starting from the latest one, that will update the recovery -/// with the all missed updates. If the WAL chain cannot be created, replica is -/// behind by a lot, so we use the regular recovery process, we send the latest -/// snapshot and all the necessary WAL files, starting from the newest WAL that -/// contains a timestamp before the snapshot. If we registered the existence of -/// the current WAL, we add the sequence number we read from it to the recovery -/// process. After all the other steps are finished, if the current WAL contains -/// the same sequence number, it's the same WAL we read while fetching the -/// recovery steps, so we can safely send it to the replica. -/// We assume that the property of preserving at least 1 WAL before the snapshot -/// is satisfied as we extract the timestamp information from it. -std::vector InMemoryReplicationClient::GetRecoverySteps( - const uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker) { - // First check if we can recover using the current wal file only - // otherwise save the seq_num of the current wal file - // This lock is also necessary to force the missed transaction to finish. - std::optional current_wal_seq_num; - std::optional current_wal_from_timestamp; - auto *storage = static_cast(storage_); - if (std::unique_lock transtacion_guard(storage->engine_lock_); storage->wal_file_) { - current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); - current_wal_from_timestamp.emplace(storage->wal_file_->FromTimestamp()); - } - - auto locker_acc = file_locker->Access(); - auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); - MG_ASSERT(wal_files, "Wal files could not be loaded"); - - auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); - std::optional latest_snapshot; - if (!snapshot_files.empty()) { - std::sort(snapshot_files.begin(), snapshot_files.end()); - latest_snapshot.emplace(std::move(snapshot_files.back())); - } - - std::vector recovery_steps; - - // No finalized WAL files were found. This means the difference is contained - // inside the current WAL or the snapshot. - if (wal_files->empty()) { - if (current_wal_from_timestamp && replica_commit >= *current_wal_from_timestamp) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Without the finalized WAL containing the current timestamp of replica, - // we cannot know if the difference is only in the current WAL or we need - // to send the snapshot. - if (latest_snapshot) { - const auto lock_success = locker_acc.AddPath(latest_snapshot->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - } - // if there are no finalized WAL files, snapshot left the current WAL - // as the WAL file containing a transaction before snapshot creation - // so we can be sure that the current WAL is present - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - // Find the longest chain of WALs for recovery. - // The chain consists ONLY of sequential WALs. - auto rwal_it = wal_files->rbegin(); - - // if the last finalized WAL is before the replica commit - // then we can recovery only from current WAL - if (rwal_it->to_timestamp <= replica_commit) { - MG_ASSERT(current_wal_seq_num); - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - return recovery_steps; - } - - uint64_t previous_seq_num{rwal_it->seq_num}; - for (; rwal_it != wal_files->rend(); ++rwal_it) { - // If the difference between two consecutive wal files is not 0 or 1 - // we have a missing WAL in our chain - if (previous_seq_num - rwal_it->seq_num > 1) { - break; - } - - // Find first WAL that contains up to replica commit, i.e. WAL - // that is before the replica commit or conatins the replica commit - // as the last committed transaction OR we managed to find the first WAL - // file. - if (replica_commit >= rwal_it->from_timestamp || rwal_it->seq_num == 0) { - if (replica_commit >= rwal_it->to_timestamp) { - // We want the WAL after because the replica already contains all the - // commits from this WAL - --rwal_it; - } - std::vector wal_chain; - auto distance_from_first = std::distance(rwal_it, wal_files->rend() - 1); - // We have managed to create WAL chain - // We need to lock these files and add them to the chain - for (auto result_wal_it = wal_files->begin() + distance_from_first; result_wal_it != wal_files->end(); - ++result_wal_it) { - const auto lock_success = locker_acc.AddPath(result_wal_it->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - wal_chain.push_back(std::move(result_wal_it->path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(wal_chain)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - return recovery_steps; - } - - previous_seq_num = rwal_it->seq_num; - } - - MG_ASSERT(latest_snapshot, "Invalid durability state, missing snapshot"); - // We didn't manage to find a WAL chain, we need to send the latest snapshot - // with its WALs - const auto lock_success = locker_acc.AddPath(latest_snapshot->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(latest_snapshot->path)); - - std::vector recovery_wal_files; - auto wal_it = wal_files->begin(); - for (; wal_it != wal_files->end(); ++wal_it) { - // Assuming recovery process is correct the snashpot should - // always retain a single WAL that contains a transaction - // before its creation - if (latest_snapshot->start_timestamp < wal_it->to_timestamp) { - if (latest_snapshot->start_timestamp < wal_it->from_timestamp) { - MG_ASSERT(wal_it != wal_files->begin(), "Invalid durability files state"); - --wal_it; - } - break; - } - } - - for (; wal_it != wal_files->end(); ++wal_it) { - const auto lock_success = locker_acc.AddPath(wal_it->path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_wal_files.push_back(std::move(wal_it->path)); - } - - // We only have a WAL before the snapshot - if (recovery_wal_files.empty()) { - const auto lock_success = locker_acc.AddPath(wal_files->back().path); - MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path."); - recovery_wal_files.push_back(std::move(wal_files->back().path)); - } - - recovery_steps.emplace_back(std::in_place_type_t{}, std::move(recovery_wal_files)); - - if (current_wal_seq_num) { - recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num}); - } - - return recovery_steps; -} - -} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.hpp b/src/storage/v2/inmemory/replication/replication_client.hpp deleted file mode 100644 index e956838e7..000000000 --- a/src/storage/v2/inmemory/replication/replication_client.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. -#pragma once - -#include "storage/v2/replication/replication_client.hpp" - -namespace memgraph::storage { - -class InMemoryStorage; - -class InMemoryReplicationClient : public ReplicationClient { - public: - InMemoryReplicationClient(InMemoryStorage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch); - - protected: - void RecoverReplica(uint64_t replica_commit) override; - - // TODO: move the GetRecoverySteps stuff below as an internal detail - using RecoverySnapshot = std::filesystem::path; - using RecoveryWals = std::vector; - struct RecoveryCurrentWal { - explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num) : current_wal_seq_num(current_wal_seq_num) {} - uint64_t current_wal_seq_num; - }; - using RecoveryStep = std::variant; - std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker); -}; - -} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 9f00081f6..c27018d24 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -18,7 +18,7 @@ /// REPLICATION /// #include "dbms/inmemory/replication_handlers.hpp" -#include "storage/v2/inmemory/replication/replication_client.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" #include "utils/resource_lock.hpp" #include "utils/stat.hpp" @@ -1608,7 +1608,7 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction // A single transaction will always be contained in a single WAL file. auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire); - repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber(), this); auto append_deltas = [&](auto callback) { // Helper lambda that traverses the delta chain on order to find the first @@ -1767,7 +1767,7 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp, this); } bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, uint64_t final_commit_timestamp) { @@ -1775,7 +1775,7 @@ bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, return true; } - repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber(), this); for (const auto &md_delta : transaction.md_deltas) { switch (md_delta.action) { @@ -1846,7 +1846,7 @@ bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp, this); } void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, @@ -1972,12 +1972,6 @@ utils::FileRetainer::FileLockerAccessor::ret_type InMemoryStorage::UnlockPath() return true; } -auto InMemoryStorage::CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr { - return std::make_unique(this, config, current_epoch); -} - std::unique_ptr InMemoryStorage::Access(std::optional override_isolation_level, bool is_main) { return std::unique_ptr(new InMemoryAccessor{Storage::Accessor::shared_access, this, diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index bfb445332..3c50326b2 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -18,10 +18,13 @@ #include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/label_index.hpp" #include "storage/v2/inmemory/label_property_index.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" +#include "storage/v2/replication/replication_client.hpp" #include "storage/v2/storage.hpp" /// REPLICATION /// #include "replication/config.hpp" +#include "storage/v2/inmemory/replication/recovery.hpp" #include "storage/v2/replication/enums.hpp" #include "storage/v2/replication/replication_storage_state.hpp" #include "storage/v2/replication/rpc.hpp" @@ -44,7 +47,10 @@ namespace memgraph::storage { class InMemoryStorage final : public Storage { friend class memgraph::dbms::InMemoryReplicationHandlers; - friend class InMemoryReplicationClient; + friend class ReplicationStorageClient; + friend std::vector GetRecoverySteps(uint64_t replica_commit, + utils::FileRetainer::FileLocker *file_locker, + const InMemoryStorage *storage); public: enum class CreateSnapshotError : uint8_t { DisabledForReplica, ReachedMaxNumTries }; @@ -332,10 +338,6 @@ class InMemoryStorage final : public Storage { using Storage::CreateTransaction; Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode, bool is_main) override; - auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr override; - void SetStorageMode(StorageMode storage_mode); private: diff --git a/src/storage/v2/replication/enums.hpp b/src/storage/v2/replication/enums.hpp index e89a1fdd3..be16ca192 100644 --- a/src/storage/v2/replication/enums.hpp +++ b/src/storage/v2/replication/enums.hpp @@ -14,6 +14,6 @@ namespace memgraph::storage::replication { -enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID }; +enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, MAYBE_BEHIND }; } // namespace memgraph::storage::replication diff --git a/src/storage/v2/replication/recovery.hpp b/src/storage/v2/replication/recovery.hpp new file mode 100644 index 000000000..346e03ecd --- /dev/null +++ b/src/storage/v2/replication/recovery.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +namespace memgraph::storage { + +using RecoverySnapshot = std::filesystem::path; +using RecoveryWals = std::vector; +struct RecoveryCurrentWal { + explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num) : current_wal_seq_num(current_wal_seq_num) {} + uint64_t current_wal_seq_num; +}; +using RecoveryStep = std::variant; + +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_client.cpp b/src/storage/v2/replication/replication_client.cpp index 33313b130..3bc1b3d32 100644 --- a/src/storage/v2/replication/replication_client.cpp +++ b/src/storage/v2/replication/replication_client.cpp @@ -9,53 +9,37 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -#include "storage/v2/replication/replication_client.hpp" +#include "replication/replication_client.hpp" +#include "storage/v2/durability/durability.hpp" +#include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/storage.hpp" +#include "utils/exceptions.hpp" +#include "utils/variant_helpers.hpp" #include #include -#include "storage/v2/durability/durability.hpp" -#include "storage/v2/storage.hpp" +namespace { +template +[[maybe_unused]] inline constexpr bool always_false_v = false; +} // namespace namespace memgraph::storage { -static auto CreateClientContext(const memgraph::replication::ReplicationClientConfig &config) - -> communication::ClientContext { - return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file} - : communication::ClientContext{}; -} +ReplicationStorageClient::ReplicationStorageClient(::memgraph::replication::ReplicationClient &client) + : client_{client} {} -ReplicationClient::ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch) - : name_{config.name}, - rpc_context_{CreateClientContext(config)}, - rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port), - &rpc_context_}, - replica_check_frequency_{config.replica_check_frequency}, - mode_{config.mode}, - storage_{storage}, - repl_epoch_{epoch} {} - -ReplicationClient::~ReplicationClient() { - auto endpoint = rpc_client_.Endpoint(); - spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port); - thread_pool_.Shutdown(); -} - -uint64_t ReplicationClient::LastCommitTimestamp() const { - return storage_->repl_storage_state_.last_commit_timestamp_.load(); -} - -void ReplicationClient::InitializeClient() { +void ReplicationStorageClient::CheckReplicaState(Storage *storage) { uint64_t current_commit_timestamp{kTimestampInitialId}; - auto stream{rpc_client_.Stream( - storage_->id(), storage_->repl_storage_state_.last_commit_timestamp_, std::string{repl_epoch_->id()})}; + auto &replStorageState = storage->repl_storage_state_; + auto stream{client_.rpc_client_.Stream( + storage->id(), replStorageState.last_commit_timestamp_, std::string{replStorageState.epoch_.id()})}; const auto replica = stream.AwaitResponse(); std::optional branching_point; - if (replica.epoch_id != repl_epoch_->id() && replica.current_commit_timestamp != kTimestampInitialId) { - auto const &history = storage_->repl_storage_state_.history; + if (replica.epoch_id != replStorageState.epoch_.id() && replica.current_commit_timestamp != kTimestampInitialId) { + auto const &history = replStorageState.history; const auto epoch_info_iter = std::find_if(history.crbegin(), history.crend(), [&](const auto &main_epoch_info) { return main_epoch_info.first == replica.epoch_id; }); @@ -71,94 +55,86 @@ void ReplicationClient::InitializeClient() { "Replica {} acted as the Main instance. Both the Main and Replica {} " "now hold unique data. Please resolve data conflicts and start the " "replication on a clean instance.", - name_, name_, name_); + client_.name_, client_.name_, client_.name_); + // State not updated, hence in MAYBE_BEHIND state return; } current_commit_timestamp = replica.current_commit_timestamp; - spdlog::trace("Current timestamp on replica {}: {}", name_, current_commit_timestamp); - spdlog::trace("Current timestamp on main: {}", storage_->repl_storage_state_.last_commit_timestamp_.load()); - if (current_commit_timestamp == storage_->repl_storage_state_.last_commit_timestamp_.load()) { - spdlog::debug("Replica '{}' up to date", name_); - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::READY); - } else { - spdlog::debug("Replica '{}' is behind", name_); - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::RECOVERY); + spdlog::trace("Current timestamp on replica {}: {}", client_.name_, current_commit_timestamp); + spdlog::trace("Current timestamp on main: {}", replStorageState.last_commit_timestamp_.load()); + replica_state_.WithLock([&](auto &state) { + if (current_commit_timestamp == replStorageState.last_commit_timestamp_.load()) { + spdlog::debug("Replica '{}' up to date", client_.name_); + state = replication::ReplicaState::READY; + } else { + spdlog::debug("Replica '{}' is behind", client_.name_); + state = replication::ReplicaState::RECOVERY; + client_.thread_pool_.AddTask( + [storage, current_commit_timestamp, this] { this->RecoverReplica(current_commit_timestamp, storage); }); } - thread_pool_.AddTask([=, this] { this->RecoverReplica(current_commit_timestamp); }); - } + }); } -TimestampInfo ReplicationClient::GetTimestampInfo() { +TimestampInfo ReplicationStorageClient::GetTimestampInfo(Storage const *storage) { TimestampInfo info; info.current_timestamp_of_replica = 0; info.current_number_of_timestamp_behind_master = 0; try { - auto stream{rpc_client_.Stream(storage_->id())}; + auto stream{client_.rpc_client_.Stream(storage->id())}; const auto response = stream.AwaitResponse(); const auto is_success = response.success; if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); } - auto main_time_stamp = storage_->repl_storage_state_.last_commit_timestamp_.load(); + auto main_time_stamp = storage->repl_storage_state_.last_commit_timestamp_.load(); info.current_timestamp_of_replica = response.current_commit_timestamp; info.current_number_of_timestamp_behind_master = response.current_commit_timestamp - main_time_stamp; } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably won't - // block + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably + // won't block } return info; } -void ReplicationClient::HandleRpcFailure() { - spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication")); - TryInitializeClientAsync(); +void ReplicationStorageClient::LogRpcFailure() { + spdlog::error( + utils::MessageWithLink("Couldn't replicate data to {}.", client_.name_, "https://memgr.ph/replication")); } -void ReplicationClient::TryInitializeClientAsync() { - thread_pool_.AddTask([this] { - rpc_client_.Abort(); - this->TryInitializeClientSync(); - }); +void ReplicationStorageClient::TryCheckReplicaStateAsync(Storage *storage) { + client_.thread_pool_.AddTask([storage, this] { this->TryCheckReplicaStateSync(storage); }); } -void ReplicationClient::TryInitializeClientSync() { +void ReplicationStorageClient::TryCheckReplicaStateSync(Storage *storage) { try { - InitializeClient(); + CheckReplicaState(storage); } catch (const rpc::VersionMismatchRpcFailedException &) { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); spdlog::error( utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}. Because the replica " "deployed is not a compatible version.", - name_, rpc_client_.Endpoint(), "https://memgr.ph/replication")); + client_.name_, client_.rpc_client_.Endpoint(), "https://memgr.ph/replication")); } catch (const rpc::RpcFailedException &) { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_, - rpc_client_.Endpoint(), "https://memgr.ph/replication")); + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", client_.name_, + client_.rpc_client_.Endpoint(), "https://memgr.ph/replication")); } } -void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) { - std::unique_lock guard(client_lock_); - const auto status = replica_state_.load(); - switch (status) { - case replication::ReplicaState::RECOVERY: - spdlog::debug("Replica {} is behind MAIN instance", name_); +void ReplicationStorageClient::StartTransactionReplication(const uint64_t current_wal_seq_num, Storage *storage) { + auto locked_state = replica_state_.Lock(); + switch (*locked_state) { + using enum replication::ReplicaState; + case RECOVERY: + spdlog::debug("Replica {} is behind MAIN instance", client_.name_); return; - case replication::ReplicaState::REPLICATING: - spdlog::debug("Replica {} missed a transaction", name_); + case REPLICATING: + spdlog::debug("Replica {} missed a transaction", client_.name_); // We missed a transaction because we're still replicating // the previous transaction so we need to go to RECOVERY // state to catch up with the missing transaction @@ -166,143 +142,169 @@ void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_s // an error can happen while we're replicating the previous // transaction after which the client should go to // INVALID state before starting the recovery process - replica_state_.store(replication::ReplicaState::RECOVERY); + // + // This is a signal to any async streams that are still finalizing to start recovery, since this commit will be + // missed. + *locked_state = RECOVERY; return; - case replication::ReplicaState::INVALID: - HandleRpcFailure(); + case MAYBE_BEHIND: + spdlog::error( + utils::MessageWithLink("Couldn't replicate data to {}.", client_.name_, "https://memgr.ph/replication")); + TryCheckReplicaStateAsync(storage); return; - case replication::ReplicaState::READY: + case READY: MG_ASSERT(!replica_stream_); try { - replica_stream_.emplace( - ReplicaStream{this, storage_->repl_storage_state_.last_commit_timestamp_.load(), current_wal_seq_num}); - replica_state_.store(replication::ReplicaState::REPLICATING); + replica_stream_.emplace(storage, client_.rpc_client_, current_wal_seq_num); + *locked_state = REPLICATING; } catch (const rpc::RpcFailedException &) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); + *locked_state = MAYBE_BEHIND; + LogRpcFailure(); } return; } } -bool ReplicationClient::FinalizeTransactionReplication() { +bool ReplicationStorageClient::FinalizeTransactionReplication(Storage *storage) { // We can only check the state because it guarantees to be only // valid during a single transaction replication (if the assumption // that this and other transaction replication functions can only be // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { + if (State() != replication::ReplicaState::REPLICATING) { return false; } - auto task = [this]() { + if (replica_stream_->IsDefunct()) return false; + + auto task = [storage, this]() { MG_ASSERT(replica_stream_, "Missing stream for transaction deltas"); try { auto response = replica_stream_->Finalize(); - replica_stream_.reset(); - std::unique_lock client_guard(client_lock_); - if (!response.success || replica_state_ == replication::ReplicaState::RECOVERY) { - replica_state_.store(replication::ReplicaState::RECOVERY); - thread_pool_.AddTask([&, this] { this->RecoverReplica(response.current_commit_timestamp); }); - } else { - replica_state_.store(replication::ReplicaState::READY); + return replica_state_.WithLock([storage, &response, this](auto &state) { + replica_stream_.reset(); + if (!response.success || state == replication::ReplicaState::RECOVERY) { + state = replication::ReplicaState::RECOVERY; + client_.thread_pool_.AddTask( + [storage, &response, this] { this->RecoverReplica(response.current_commit_timestamp, storage); }); + return false; + } + state = replication::ReplicaState::READY; return true; - } + }); } catch (const rpc::RpcFailedException &) { - replica_stream_.reset(); - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); + replica_state_.WithLock([this](auto &state) { + replica_stream_.reset(); + state = replication::ReplicaState::MAYBE_BEHIND; + }); + LogRpcFailure(); + return false; } - return false; }; - if (mode_ == memgraph::replication::ReplicationMode::ASYNC) { - thread_pool_.AddTask([=] { (void)task(); }); + if (client_.mode_ == memgraph::replication::ReplicationMode::ASYNC) { + client_.thread_pool_.AddTask([task = std::move(task)] { (void)task(); }); return true; } return task(); } -void ReplicationClient::FrequentCheck() { - const auto is_success = std::invoke([this]() { - try { - auto stream{rpc_client_.Stream()}; - const auto response = stream.AwaitResponse(); - return response.success; - } catch (const rpc::RpcFailedException &) { - return false; - } - }); - // States: READY, REPLICATING, RECOVERY, INVALID - // If success && ready, replicating, recovery -> stay the same because something good is going on. - // If success && INVALID -> [it's possible that replica came back to life] -> TryInitializeClient. - // If fail -> [replica is not reachable at all] -> INVALID state. - // NOTE: TryInitializeClient might return nothing if there is a branching point. - // NOTE: The early return pattern simplified the code, but the behavior should be as explained. - if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - return; - } - if (replica_state_.load() == replication::ReplicaState::INVALID) { - TryInitializeClientAsync(); - } +void ReplicationStorageClient::Start(Storage *storage) { + spdlog::trace("Replication client started for database \"{}\"", storage->id()); + TryCheckReplicaStateSync(storage); } -void ReplicationClient::Start() { - auto const &endpoint = rpc_client_.Endpoint(); - spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); - - TryInitializeClientSync(); - - // Help the user to get the most accurate replica state possible. - if (replica_check_frequency_ > std::chrono::seconds(0)) { - replica_checker_.Run("Replica Checker", replica_check_frequency_, [this] { this->FrequentCheck(); }); +void ReplicationStorageClient::RecoverReplica(uint64_t replica_commit, memgraph::storage::Storage *storage) { + if (storage->storage_mode_ != StorageMode::IN_MEMORY_TRANSACTIONAL) { + throw utils::BasicException("Only InMemoryTransactional mode supports replication!"); } -} + spdlog::debug("Starting replica recovery"); + auto *mem_storage = static_cast(storage); -void ReplicationClient::IfStreamingTransaction(const std::function &callback) { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return; - } + while (true) { + auto file_locker = mem_storage->file_retainer_.AddLocker(); - try { - callback(*replica_stream_); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); + const auto steps = GetRecoverySteps(replica_commit, &file_locker, mem_storage); + int i = 0; + for (const RecoveryStep &recovery_step : steps) { + spdlog::trace("Recovering in step: {}", i++); + try { + rpc::Client &rpcClient = client_.rpc_client_; + std::visit(utils::Overloaded{ + [&replica_commit, mem_storage, &rpcClient](RecoverySnapshot const &snapshot) { + spdlog::debug("Sending the latest snapshot file: {}", snapshot); + auto response = TransferSnapshot(mem_storage->id(), rpcClient, snapshot); + replica_commit = response.current_commit_timestamp; + }, + [&replica_commit, mem_storage, &rpcClient](RecoveryWals const &wals) { + spdlog::debug("Sending the latest wal files"); + auto response = TransferWalFiles(mem_storage->id(), rpcClient, wals); + replica_commit = response.current_commit_timestamp; + spdlog::debug("Wal files successfully transferred."); + }, + [&replica_commit, mem_storage, &rpcClient](RecoveryCurrentWal const ¤t_wal) { + std::unique_lock transaction_guard(mem_storage->engine_lock_); + if (mem_storage->wal_file_ && + mem_storage->wal_file_->SequenceNumber() == current_wal.current_wal_seq_num) { + mem_storage->wal_file_->DisableFlushing(); + transaction_guard.unlock(); + spdlog::debug("Sending current wal file"); + replica_commit = ReplicateCurrentWal(mem_storage, rpcClient, *mem_storage->wal_file_); + mem_storage->wal_file_->EnableFlushing(); + } else { + spdlog::debug("Cannot recover using current wal file"); + } + }, + [](auto const &in) { + static_assert(always_false_v, "Missing type from variant visitor"); + }, + }, + recovery_step); + } catch (const rpc::RpcFailedException &) { + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); + return; + } + } + + spdlog::trace("Current timestamp on replica: {}", replica_commit); + // To avoid the situation where we read a correct commit timestamp in + // one thread, and after that another thread commits a different a + // transaction and THEN we set the state to READY in the first thread, + // we set this lock before checking the timestamp. + // We will detect that the state is invalid during the next commit, + // because replication::AppendDeltasRpc sends the last commit timestamp which + // replica checks if it's the same last commit timestamp it received + // and we will go to recovery. + // By adding this lock, we can avoid that, and go to RECOVERY immediately. + const auto last_commit_timestamp = storage->repl_storage_state_.last_commit_timestamp_.load(); + SPDLOG_INFO("Replica timestamp: {}", replica_commit); + SPDLOG_INFO("Last commit: {}", last_commit_timestamp); + if (last_commit_timestamp == replica_commit) { + replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::READY; }); + return; } - HandleRpcFailure(); } } ////// ReplicaStream ////// -ReplicaStream::ReplicaStream(ReplicationClient *self, const uint64_t previous_commit_timestamp, - const uint64_t current_seq_num) - : self_(self), - stream_(self_->rpc_client_.Stream(self->GetStorageId(), previous_commit_timestamp, - current_seq_num)) { +ReplicaStream::ReplicaStream(Storage *storage, rpc::Client &rpc_client, const uint64_t current_seq_num) + : storage_{storage}, + stream_(rpc_client.Stream( + storage->id(), storage->repl_storage_state_.last_commit_timestamp_.load(), current_seq_num)) { replication::Encoder encoder{stream_.GetBuilder()}; - - encoder.WriteString(self->repl_epoch_->id()); + encoder.WriteString(storage->repl_storage_state_.epoch_.id()); } void ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - auto *storage = self_->GetStorage(); - EncodeDelta(&encoder, storage->name_id_mapper_.get(), storage->config_.items, delta, vertex, final_commit_timestamp); + EncodeDelta(&encoder, storage_->name_id_mapper_.get(), storage_->config_.items, delta, vertex, + final_commit_timestamp); } void ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge, uint64_t final_commit_timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - EncodeDelta(&encoder, self_->GetStorage()->name_id_mapper_.get(), delta, edge, final_commit_timestamp); + EncodeDelta(&encoder, storage_->name_id_mapper_.get(), delta, edge, final_commit_timestamp); } void ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) { @@ -314,11 +316,10 @@ void ReplicaStream::AppendOperation(durability::StorageMetadataOperation operati const std::set &properties, const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - EncodeOperation(&encoder, self_->GetStorage()->name_id_mapper_.get(), operation, label, properties, stats, - property_stats, timestamp); + EncodeOperation(&encoder, storage_->name_id_mapper_.get(), operation, label, properties, stats, property_stats, + timestamp); } replication::AppendDeltasRes ReplicaStream::Finalize() { return stream_.AwaitResponse(); } -auto ReplicationClient::GetStorageId() const -> std::string { return storage_->id(); } } // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index 8cd8cb384..3d2c019e9 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -13,6 +13,8 @@ #include "replication/config.hpp" #include "replication/epoch.hpp" +#include "replication/messages.hpp" +#include "replication/replication_client.hpp" #include "rpc/client.hpp" #include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/id_types.hpp" @@ -23,9 +25,12 @@ #include "storage/v2/replication/rpc.hpp" #include "utils/file_locker.hpp" #include "utils/scheduler.hpp" +#include "utils/synchronized.hpp" #include "utils/thread_pool.hpp" #include +#include +#include #include #include #include @@ -37,12 +42,12 @@ struct Delta; struct Vertex; struct Edge; class Storage; -class ReplicationClient; +class ReplicationStorageClient; // Handler used for transferring the current transaction. class ReplicaStream { public: - explicit ReplicaStream(ReplicationClient *self, uint64_t previous_commit_timestamp, uint64_t current_seq_num); + explicit ReplicaStream(Storage *storage, rpc::Client &rpc_client, uint64_t current_seq_num); /// @throw rpc::RpcFailedException void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp); @@ -61,85 +66,84 @@ class ReplicaStream { /// @throw rpc::RpcFailedException replication::AppendDeltasRes Finalize(); + bool IsDefunct() const { return stream_.IsDefunct(); } + private: - ReplicationClient *self_; + Storage *storage_; rpc::Client::StreamHandler stream_; }; -class ReplicationClient { - friend class CurrentWalHandler; +template +concept InvocableWithStream = std::invocable; + +// TODO Rename to something without the word "client" +class ReplicationStorageClient { + friend class InMemoryCurrentWalHandler; friend class ReplicaStream; + friend struct ::memgraph::replication::ReplicationClient; public: - ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *epoch); + explicit ReplicationStorageClient(::memgraph::replication::ReplicationClient &client); - ReplicationClient(ReplicationClient const &) = delete; - ReplicationClient &operator=(ReplicationClient const &) = delete; - ReplicationClient(ReplicationClient &&) noexcept = delete; - ReplicationClient &operator=(ReplicationClient &&) noexcept = delete; + ReplicationStorageClient(ReplicationStorageClient const &) = delete; + ReplicationStorageClient &operator=(ReplicationStorageClient const &) = delete; + ReplicationStorageClient(ReplicationStorageClient &&) noexcept = delete; + ReplicationStorageClient &operator=(ReplicationStorageClient &&) noexcept = delete; - virtual ~ReplicationClient(); + ~ReplicationStorageClient() = default; - auto Mode() const -> memgraph::replication::ReplicationMode { return mode_; } - auto Name() const -> std::string const & { return name_; } - auto Endpoint() const -> io::network::Endpoint const & { return rpc_client_.Endpoint(); } - auto State() const -> replication::ReplicaState { return replica_state_.load(); } - auto GetTimestampInfo() -> TimestampInfo; + // TODO Remove the client related functions + auto Mode() const -> memgraph::replication::ReplicationMode { return client_.mode_; } + auto Name() const -> std::string const & { return client_.name_; } + auto Endpoint() const -> io::network::Endpoint const & { return client_.rpc_client_.Endpoint(); } - auto GetStorageId() const -> std::string; + auto State() const -> replication::ReplicaState { return replica_state_.WithLock(std::identity()); } + auto GetTimestampInfo(Storage const *storage) -> TimestampInfo; + + void Start(Storage *storage); + void StartTransactionReplication(uint64_t current_wal_seq_num, Storage *storage); - void Start(); - void StartTransactionReplication(uint64_t current_wal_seq_num); // Replication clients can be removed at any point // so to avoid any complexity of checking if the client was removed whenever // we want to send part of transaction and to avoid adding some GC logic this // function will run a callback if, after previously callling // StartTransactionReplication, stream is created. - void IfStreamingTransaction(const std::function &callback); + template + void IfStreamingTransaction(F &&callback) { + // We can only check the state because it guarantees to be only + // valid during a single transaction replication (if the assumption + // that this and other transaction replication functions can only be + // called from a one thread stands) + if (State() != replication::ReplicaState::REPLICATING) { + return; + } + if (replica_stream_->IsDefunct()) return; + try { + callback(*replica_stream_); // failure state what if not streaming (std::nullopt) + } catch (const rpc::RpcFailedException &) { + return replica_state_.WithLock([](auto &state) { state = replication::ReplicaState::MAYBE_BEHIND; }); + LogRpcFailure(); + } + } + // Return whether the transaction could be finalized on the replication client or not. - [[nodiscard]] bool FinalizeTransactionReplication(); + [[nodiscard]] bool FinalizeTransactionReplication(Storage *storage); - protected: - virtual void RecoverReplica(uint64_t replica_commit) = 0; + void TryCheckReplicaStateAsync(Storage *storage); // TODO Move back to private + private: + void RecoverReplica(uint64_t replica_commit, memgraph::storage::Storage *storage); - auto GetStorage() -> Storage * { return storage_; } - auto LastCommitTimestamp() const -> uint64_t; - void InitializeClient(); - void HandleRpcFailure(); - void TryInitializeClientAsync(); - void TryInitializeClientSync(); - void FrequentCheck(); + void CheckReplicaState(Storage *storage); + void LogRpcFailure(); + void TryCheckReplicaStateSync(Storage *storage); + void FrequentCheck(Storage *storage); - std::string name_; - communication::ClientContext rpc_context_; - rpc::Client rpc_client_; - std::chrono::seconds replica_check_frequency_; - - std::optional replica_stream_; - memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC}; - - utils::SpinLock client_lock_; - // This thread pool is used for background tasks so we don't - // block the main storage thread - // We use only 1 thread for 2 reasons: - // - background tasks ALWAYS contain some kind of RPC communication. - // We can't have multiple RPC communication from a same client - // because that's not logically valid (e.g. you cannot send a snapshot - // and WAL at a same time because WAL will arrive earlier and be applied - // before the snapshot which is not correct) - // - the implementation is simplified as we have a total control of what - // this pool is executing. Also, we can simply queue multiple tasks - // and be sure of the execution order. - // Not having mulitple possible threads in the same client allows us - // to ignore concurrency problems inside the client. - utils::ThreadPool thread_pool_{1}; - std::atomic replica_state_{replication::ReplicaState::INVALID}; - - utils::Scheduler replica_checker_; - Storage *storage_; - - memgraph::replication::ReplicationEpoch const *repl_epoch_; + ::memgraph::replication::ReplicationClient &client_; + // TODO Do not store the stream, make is a local variable + std::optional + replica_stream_; // Currently active stream (nullopt if not in use), note: a single stream per rpc client + mutable utils::Synchronized replica_state_{ + replication::ReplicaState::MAYBE_BEHIND}; }; } // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_storage_state.cpp b/src/storage/v2/replication/replication_storage_state.cpp index 1cd0bec09..a443c7171 100644 --- a/src/storage/v2/replication/replication_storage_state.cpp +++ b/src/storage/v2/replication/replication_storage_state.cpp @@ -16,10 +16,10 @@ namespace memgraph::storage { -void ReplicationStorageState::InitializeTransaction(uint64_t seq_num) { - replication_clients_.WithLock([&](auto &clients) { +void ReplicationStorageState::InitializeTransaction(uint64_t seq_num, Storage *storage) { + replication_clients_.WithLock([=](auto &clients) { for (auto &client : clients) { - client->StartTransactionReplication(seq_num); + client->StartTransactionReplication(seq_num, storage); } }); } @@ -52,12 +52,12 @@ void ReplicationStorageState::AppendOperation(durability::StorageMetadataOperati }); } -bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp) { +bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp, Storage *storage) { return replication_clients_.WithLock([=](auto &clients) { bool finalized_on_all_replicas = true; for (ReplicationClientPtr &client : clients) { client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(timestamp); }); - const auto finalized = client->FinalizeTransactionReplication(); + const auto finalized = client->FinalizeTransactionReplication(storage); if (client->Mode() == memgraph::replication::ReplicationMode::SYNC) { finalized_on_all_replicas = finalized && finalized_on_all_replicas; @@ -78,12 +78,12 @@ std::optional ReplicationStorageState::GetReplicaStat }); } -std::vector ReplicationStorageState::ReplicasInfo() const { - return replication_clients_.WithReadLock([](auto const &clients) { +std::vector ReplicationStorageState::ReplicasInfo(const Storage *storage) const { + return replication_clients_.WithReadLock([storage](auto const &clients) { std::vector replica_infos; replica_infos.reserve(clients.size()); - auto const asReplicaInfo = [](ReplicationClientPtr const &client) -> ReplicaInfo { - return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo()}; + auto const asReplicaInfo = [storage](ReplicationClientPtr const &client) -> ReplicaInfo { + return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo(storage)}; }; std::transform(clients.begin(), clients.end(), std::back_inserter(replica_infos), asReplicaInfo); return replica_infos; diff --git a/src/storage/v2/replication/replication_storage_state.hpp b/src/storage/v2/replication/replication_storage_state.hpp index afedb3950..e3d6b94a0 100644 --- a/src/storage/v2/replication/replication_storage_state.hpp +++ b/src/storage/v2/replication/replication_storage_state.hpp @@ -12,11 +12,13 @@ #pragma once #include +#include #include "kvstore/kvstore.hpp" #include "storage/v2/delta.hpp" #include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/transaction.hpp" +#include "utils/exceptions.hpp" #include "utils/result.hpp" /// REPLICATION /// @@ -33,21 +35,21 @@ namespace memgraph::storage { class Storage; -class ReplicationClient; +class ReplicationStorageClient; struct ReplicationStorageState { // Only MAIN can send - void InitializeTransaction(uint64_t seq_num); + void InitializeTransaction(uint64_t seq_num, Storage *storage); void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp); void AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp); void AppendOperation(durability::StorageMetadataOperation operation, LabelId label, const std::set &properties, const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t final_commit_timestamp); - bool FinalizeTransaction(uint64_t timestamp); + bool FinalizeTransaction(uint64_t timestamp, Storage *storage); // Getters auto GetReplicaState(std::string_view name) const -> std::optional; - auto ReplicasInfo() const -> std::vector; + auto ReplicasInfo(const Storage *storage) const -> std::vector; // History void TrackLatestHistory(); @@ -55,6 +57,19 @@ struct ReplicationStorageState { void Reset(); + template + bool WithClient(std::string_view replica_name, F &&callback) { + return replication_clients_.WithLock([replica_name, cb = std::forward(callback)](auto &clients) { + for (const auto &client : clients) { + if (client->Name() == replica_name) { + cb(client.get()); + return true; + } + } + return false; + }); + } + // Questions: // - storage durability <- databases/*name*/wal and snapshots (where this for epoch_id) // - multi-tenant durability <- databases/.durability (there is a list of all active tenants) @@ -74,7 +89,7 @@ struct ReplicationStorageState { // This way we can initialize client in main thread which means // that we can immediately notify the user if the initialization // failed. - using ReplicationClientPtr = std::unique_ptr; + using ReplicationClientPtr = std::unique_ptr; using ReplicationClientList = utils::Synchronized, utils::RWSpinLock>; ReplicationClientList replication_clients_; diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 8d8b06cd6..4142da3ca 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -109,7 +109,7 @@ struct EdgeInfoForDeletion { class Storage { friend class ReplicationServer; - friend class ReplicationClient; + friend class ReplicationStorageClient; public: Storage(Config config, StorageMode storage_mode); @@ -355,11 +355,7 @@ class Storage { virtual void PrepareForNewEpoch() = 0; - virtual auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config, - const memgraph::replication::ReplicationEpoch *current_epoch) - -> std::unique_ptr = 0; - - auto ReplicasInfo() const { return repl_storage_state_.ReplicasInfo(); } + auto ReplicasInfo() const { return repl_storage_state_.ReplicasInfo(this); } auto GetReplicaState(std::string_view name) const -> std::optional { return repl_storage_state_.GetReplicaState(name); } @@ -384,7 +380,7 @@ class Storage { Config config_; // Transaction engine - utils::SpinLock engine_lock_; + mutable utils::SpinLock engine_lock_; uint64_t timestamp_{kTimestampInitialId}; uint64_t transaction_id_{kTransactionInitialId}; diff --git a/tests/e2e/replication/show_while_creating_invalid_state.py b/tests/e2e/replication/show_while_creating_invalid_state.py index 996955dc1..74dcbce74 100644 --- a/tests/e2e/replication/show_while_creating_invalid_state.py +++ b/tests/e2e/replication/show_while_creating_invalid_state.py @@ -123,6 +123,143 @@ def test_show_replicas(connection): assert actual_data == expected_data +def test_drop_replicas(connection): + # Goal of this test is to check the DROP REPLICAS command. + # 0/ Manually start main and all replicas + # 1/ Check status of the replicas + # 2/ Kill replica 3 + # 3/ Drop replica 3 and check status + # 4/ Stop replica 4 + # 5/ Drop replica 4 and check status + # 6/ Kill replica 1 + # 7/ Drop replica 1 and check status + # 8/ Stop replica 2 + # 9/ Drop replica 2 and check status + # 10/ Restart all replicas + # 11/ Register them + # 12/ Drop all and check status + + def retrieve_data(): + return set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + # 0/ + interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION) + + cursor = connection(7687, "main").cursor() + + # 1/ + actual_data = set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + EXPECTED_COLUMN_NAMES = { + "name", + "socket_address", + "sync_mode", + "current_timestamp_of_replica", + "number_of_timestamp_behind_master", + "state", + } + + actual_column_names = {x.name for x in cursor.description} + assert actual_column_names == EXPECTED_COLUMN_NAMES + + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 2/ + interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_3") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "invalid"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 3/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_3") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 4/ + interactive_mg_runner.stop(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_4") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "invalid"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 5/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_4") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 6/ + interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_1") + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "invalid"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 7/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_1") + expected_data = { + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 8/ + interactive_mg_runner.stop(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_2") + expected_data = { + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "invalid"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 9/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_2") + expected_data = set() + mg_sleep_and_assert(expected_data, retrieve_data) + + # 10/ + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_1") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_2") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_3") + interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "replica_4") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_3 ASYNC TO '127.0.0.1:10003';") + execute_and_fetch_all(cursor, "REGISTER REPLICA replica_4 ASYNC TO '127.0.0.1:10004';") + + # 11/ + expected_data = { + ("replica_1", "127.0.0.1:10001", "sync", 0, 0, "ready"), + ("replica_2", "127.0.0.1:10002", "sync", 0, 0, "ready"), + ("replica_3", "127.0.0.1:10003", "async", 0, 0, "ready"), + ("replica_4", "127.0.0.1:10004", "async", 0, 0, "ready"), + } + mg_sleep_and_assert(expected_data, retrieve_data) + + # 12/ + execute_and_fetch_all(cursor, "DROP REPLICA replica_1") + execute_and_fetch_all(cursor, "DROP REPLICA replica_2") + execute_and_fetch_all(cursor, "DROP REPLICA replica_3") + execute_and_fetch_all(cursor, "DROP REPLICA replica_4") + expected_data = set() + mg_sleep_and_assert(expected_data, retrieve_data) + + def test_basic_recovery(connection): # Goal of this test is to check the recovery of main. # 0/ We start all replicas manually: we want to be able to kill them ourselves without relying on external tooling to kill processes. @@ -630,10 +767,26 @@ def test_async_replication_when_main_is_killed(): ) # 2/ - for index in range(50): + # First make sure that anything has been replicated + for index in range(0, 5): + interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(f"CREATE (p:Number {{name:{index}}})") + expected_data = [("async_replica", "127.0.0.1:10001", "async", "ready")] + + def retrieve_data(): + replicas = interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query("SHOW REPLICAS;") + return [ + (replica_name, ip, mode, status) + for replica_name, ip, mode, timestamp, timestamp_behind_main, status in replicas + ] + + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + for index in range(5, 50): interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(f"CREATE (p:Number {{name:{index}}})") if random.randint(0, 100) > 95: main_killed = f"Main was killed at index={index}" + print(main_killed) interactive_mg_runner.kill(CONFIGURATION, "main") break diff --git a/tests/integration/telemetry/client.cpp b/tests/integration/telemetry/client.cpp index 558e0a6bc..34e1c2a67 100644 --- a/tests/integration/telemetry/client.cpp +++ b/tests/integration/telemetry/client.cpp @@ -41,7 +41,7 @@ int main(int argc, char **argv) { memgraph::storage::UpdatePaths(db_config, data_directory); memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(db_config)); - memgraph::dbms::DbmsHandler dbms_handler(db_config, repl_state + memgraph::dbms::DbmsHandler dbms_handler(db_config #ifdef MG_ENTERPRISE , &auth_, false, false diff --git a/tests/unit/dbms_handler.cpp b/tests/unit/dbms_handler.cpp index 75efddefe..a811e4159 100644 --- a/tests/unit/dbms_handler.cpp +++ b/tests/unit/dbms_handler.cpp @@ -52,18 +52,15 @@ class TestEnvironment : public ::testing::Environment { auth = std::make_unique>( storage_directory / "auth"); - repl_state_.emplace(memgraph::storage::ReplicationStateRootPath(storage_conf)); - ptr_ = std::make_unique(storage_conf, *repl_state_, auth.get(), false, true); + ptr_ = std::make_unique(storage_conf, auth.get(), false, true); } void TearDown() override { ptr_.reset(); auth.reset(); - repl_state_.reset(); } static std::unique_ptr ptr_; - std::optional repl_state_; }; std::unique_ptr TestEnvironment::ptr_ = nullptr; diff --git a/tests/unit/dbms_handler_community.cpp b/tests/unit/dbms_handler_community.cpp index efce2854d..3848cd347 100644 --- a/tests/unit/dbms_handler_community.cpp +++ b/tests/unit/dbms_handler_community.cpp @@ -52,18 +52,15 @@ class TestEnvironment : public ::testing::Environment { auth = std::make_unique>( storage_directory / "auth"); - repl_state_.emplace(memgraph::storage::ReplicationStateRootPath(storage_conf)); - ptr_ = std::make_unique(storage_conf, *repl_state_); + ptr_ = std::make_unique(storage_conf); } void TearDown() override { ptr_.reset(); auth.reset(); - repl_state_.reset(); } static std::unique_ptr ptr_; - std::optional repl_state_; }; std::unique_ptr TestEnvironment::ptr_ = nullptr; diff --git a/tests/unit/storage_v2_replication.cpp b/tests/unit/storage_v2_replication.cpp index 261b2ccf0..f07130c4a 100644 --- a/tests/unit/storage_v2_replication.cpp +++ b/tests/unit/storage_v2_replication.cpp @@ -102,8 +102,7 @@ class ReplicationTest : public ::testing::Test { struct MinMemgraph { MinMemgraph(const memgraph::storage::Config &conf) - : repl_state{ReplicationStateRootPath(conf)}, - dbms{conf, repl_state + : dbms{conf #ifdef MG_ENTERPRISE , reinterpret_cast< @@ -111,11 +110,12 @@ struct MinMemgraph { true, false #endif }, + repl_state{dbms.ReplicationState()}, db{*dbms.Get().get()}, - repl_handler(repl_state, dbms) { + repl_handler(dbms) { } - memgraph::replication::ReplicationState repl_state; memgraph::dbms::DbmsHandler dbms; + memgraph::replication::ReplicationState &repl_state; memgraph::dbms::Database &db; ReplicationHandler repl_handler; }; @@ -130,14 +130,13 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) { .port = ports[0], }); - ASSERT_FALSE(main.repl_handler - .RegisterReplica(ReplicationClientConfig{ - .name = "REPLICA", - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) - .HasError()); + const auto ® = main.repl_handler.RegisterReplica(ReplicationClientConfig{ + .name = "REPLICA", + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }); + ASSERT_FALSE(reg.HasError()) << (int)reg.GetError(); // vertex create // vertex add label @@ -966,14 +965,14 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAfterDroppingReplica) { .ip_address = local_host, .port = ports[0], }); - ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(res.HasError()) << (int)res.GetError(); res = main->repl_handler.RegisterReplica(ReplicationClientConfig{ .name = replicas[1], .mode = ReplicationMode::SYNC, .ip_address = local_host, .port = ports[1], }); - ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(res.HasError()) << (int)res.GetError(); auto replica_infos = main->db.storage()->ReplicasInfo(); From 70db2fca565463fe4137f98169aaf37e562d1a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= <48765171+MarkoBarisic@users.noreply.github.com> Date: Thu, 23 Nov 2023 13:46:04 +0100 Subject: [PATCH 10/10] Change package_all to package_memgraph (#1507) Add the ability to pick a specific package to build --- .github/workflows/package_all.yaml | 263 ------------------ ...ge_specific.yaml => package_memgraph.yaml} | 33 +-- 2 files changed, 17 insertions(+), 279 deletions(-) delete mode 100644 .github/workflows/package_all.yaml rename .github/workflows/{package_specific.yaml => package_memgraph.yaml} (87%) diff --git a/.github/workflows/package_all.yaml b/.github/workflows/package_all.yaml deleted file mode 100644 index f1831616a..000000000 --- a/.github/workflows/package_all.yaml +++ /dev/null @@ -1,263 +0,0 @@ -name: Package All - -# TODO(gitbuda): Cleanup docker container if GHA job was canceled. - -on: - workflow_dispatch: - inputs: - memgraph_version: - description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'" - required: false - build_type: - type: choice - description: "Memgraph Build type. Default value is Release." - default: 'Release' - options: - - Release - - RelWithDebInfo - -jobs: - centos-7: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: centos-7 - path: build/output/centos-7/memgraph*.rpm - - centos-9: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: centos-9 - path: build/output/centos-9/memgraph*.rpm - - debian-10: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-10 - path: build/output/debian-10/memgraph*.deb - - debian-11: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11 - path: build/output/debian-11/memgraph*.deb - - docker: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - cd release/package - ./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker - ./run.sh docker - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: docker - path: build/output/docker/memgraph*.tar.gz - - ubuntu-1804: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-18.04 - path: build/output/ubuntu-18.04/memgraph*.deb - - ubuntu-2004: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-20.04 - path: build/output/ubuntu-20.04/memgraph*.deb - - ubuntu-2204: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-22.04 - path: build/output/ubuntu-22.04/memgraph*.deb - - debian-11-platform: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11-platform - path: build/output/debian-11/memgraph*.deb - - fedora-36: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: fedora-36 - path: build/output/fedora-36/memgraph*.rpm - - amzn-2: - runs-on: [self-hosted, DockerMgBuild, X64] - timeout-minutes: 60 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: amzn-2 - path: build/output/amzn-2/memgraph*.rpm - - debian-11-arm: - runs-on: [self-hosted, DockerMgBuild, ARM64, strange] - timeout-minutes: 120 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: debian-11-aarch64 - path: build/output/debian-11-arm/memgraph*.deb - - ubuntu-2204-arm: - runs-on: [self-hosted, DockerMgBuild, ARM64, strange] - timeout-minutes: 120 - steps: - - name: "Set up repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Required because of release/get_version.py - - name: "Build package" - run: | - ./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }} - - name: "Upload package" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-22.04-aarch64 - path: build/output/ubuntu-22.04-arm/memgraph*.deb - - upload-to-s3: - # only run upload if we specified version. Allows for runs without upload - if: "${{ github.event.inputs.memgraph_version != '' }}" - needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm] - runs-on: ubuntu-latest - steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - # name: # if name input parameter is not provided, all artifacts are downloaded - # and put in directories named after each one. - path: build/output/release - - name: Upload to S3 - uses: jakejarvis/s3-sync-action@v0.5.1 - env: - AWS_S3_BUCKET: "download.memgraph.com" - AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "eu-west-1" - SOURCE_DIR: "build/output/release" - DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/" diff --git a/.github/workflows/package_specific.yaml b/.github/workflows/package_memgraph.yaml similarity index 87% rename from .github/workflows/package_specific.yaml rename to .github/workflows/package_memgraph.yaml index c599f65ef..48a61ca53 100644 --- a/.github/workflows/package_specific.yaml +++ b/.github/workflows/package_memgraph.yaml @@ -1,4 +1,4 @@ -name: Package Specific +name: Package memgraph # TODO(gitbuda): Cleanup docker container if GHA job was canceled. @@ -10,16 +10,17 @@ on: required: false build_type: type: choice - description: "Memgraph Build type. Default value is Release." + description: "Memgraph Build type. Default value is Release" default: 'Release' options: - Release - RelWithDebInfo target_os: type: choice - description: "Target OS for which memgraph will be packaged. Default is Ubuntu 22.04" + description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04" default: 'ubuntu-22_04' options: + - all - amzn-2 - centos-7 - centos-9 @@ -36,7 +37,7 @@ on: jobs: amzn-2: - if: ${{ github.event.inputs.target_os == 'amzn-2' }} + if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -54,7 +55,7 @@ jobs: path: build/output/amzn-2/memgraph*.rpm centos-7: - if: ${{ github.event.inputs.target_os == 'centos-7' }} + if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -72,7 +73,7 @@ jobs: path: build/output/centos-7/memgraph*.rpm centos-9: - if: ${{ github.event.inputs.target_os == 'centos-9' }} + if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -90,7 +91,7 @@ jobs: path: build/output/centos-9/memgraph*.rpm debian-10: - if: ${{ github.event.inputs.target_os == 'debian-10' }} + if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -108,7 +109,7 @@ jobs: path: build/output/debian-10/memgraph*.deb debian-11: - if: ${{ github.event.inputs.target_os == 'debian-11' }} + if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -126,7 +127,7 @@ jobs: path: build/output/debian-11/memgraph*.deb debian-11-arm: - if: ${{ github.event.inputs.target_os == 'debian-11-arm' }} + if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: @@ -144,7 +145,7 @@ jobs: path: build/output/debian-11-arm/memgraph*.deb debian-11-platform: - if: ${{ github.event.inputs.target_os == 'debian-11-platform' }} + if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -162,7 +163,7 @@ jobs: path: build/output/debian-11/memgraph*.deb docker: - if: ${{ github.event.inputs.target_os == 'docker' }} + if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -182,7 +183,7 @@ jobs: path: build/output/docker/memgraph*.tar.gz fedora-36: - if: ${{ github.event.inputs.target_os == 'fedora-36' }} + if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -200,7 +201,7 @@ jobs: path: build/output/fedora-36/memgraph*.rpm ubuntu-18_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -218,7 +219,7 @@ jobs: path: build/output/ubuntu-18.04/memgraph*.deb ubuntu-20_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -236,7 +237,7 @@ jobs: path: build/output/ubuntu-20.04/memgraph*.deb ubuntu-22_04: - if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: @@ -254,7 +255,7 @@ jobs: path: build/output/ubuntu-22.04/memgraph*.deb ubuntu-22_04-arm: - if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' }} + if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }} runs-on: [self-hosted, DockerMgBuild, ARM64, strange] timeout-minutes: 120 steps: