diff --git a/config/flags.yaml b/config/flags.yaml index fba207a4f..621d27345 100644 --- a/config/flags.yaml +++ b/config/flags.yaml @@ -99,6 +99,10 @@ modifications: value: "SNAPSHOT_ISOLATION" override: true + - name: "storage_mode" + value: "IN_MEMORY_TRANSACTIONAL" + override: true + - name: "allow_load_csv" value: "true" override: false diff --git a/src/dbms/database.cpp b/src/dbms/database.cpp index 89c3d2bf5..77d9087ef 100644 --- a/src/dbms/database.cpp +++ b/src/dbms/database.cpp @@ -10,8 +10,10 @@ // licenses/APL.txt. #include "dbms/database.hpp" +#include "flags/storage_mode.hpp" #include "storage/v2/disk/storage.hpp" #include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/storage_mode.hpp" template struct memgraph::utils::Gatekeeper; @@ -20,10 +22,11 @@ namespace memgraph::dbms { Database::Database(const storage::Config &config) : trigger_store_(config.durability.storage_directory / "triggers"), streams_{config.durability.storage_directory / "streams"} { - if (config.force_on_disk || utils::DirExists(config.disk.main_storage_directory)) { + if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk || + utils::DirExists(config.disk.main_storage_directory)) { storage_ = std::make_unique(config); } else { - storage_ = std::make_unique(config); + storage_ = std::make_unique(config, config.storage_mode); } } diff --git a/src/flags/CMakeLists.txt b/src/flags/CMakeLists.txt index 66a82fbdf..6f4c1b748 100644 --- a/src/flags/CMakeLists.txt +++ b/src/flags/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(mg-flags STATIC audit.cpp isolation_level.cpp log_level.cpp memory_limit.cpp - run_time_configurable.cpp) + run_time_configurable.cpp + storage_mode.cpp) target_include_directories(mg-flags PUBLIC ${CMAKE_SOURCE_DIR}/include) target_link_libraries(mg-flags PUBLIC spdlog::spdlog mg-settings mg-utils) diff --git a/src/flags/all.hpp b/src/flags/all.hpp index 1410d3d92..9cdcd1e8f 100644 --- a/src/flags/all.hpp +++ b/src/flags/all.hpp @@ -17,3 +17,4 @@ #include "flags/log_level.hpp" #include "flags/memory_limit.hpp" #include "flags/run_time_configurable.hpp" +#include "flags/storage_mode.hpp" diff --git a/src/flags/storage_mode.cpp b/src/flags/storage_mode.cpp new file mode 100644 index 000000000..b342719dd --- /dev/null +++ b/src/flags/storage_mode.cpp @@ -0,0 +1,55 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "flags/storage_mode.hpp" + +#include "storage/v2/storage_mode.hpp" +#include "utils/enum.hpp" +#include "utils/flag_validation.hpp" + +#include "gflags/gflags.h" + +#include + +inline constexpr std::array storage_mode_mappings{ + std::pair{std::string_view{"IN_MEMORY_TRANSACTIONAL"}, memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL}, + std::pair{std::string_view{"IN_MEMORY_ANALYTICAL"}, memgraph::storage::StorageMode::IN_MEMORY_ANALYTICAL}, + std::pair{std::string_view{"ON_DISK_TRANSACTIONAL"}, memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL}}; + +const std::string storage_mode_help_string = + fmt::format("Default storage mode Memgraph uses. Allowed values: {}", + memgraph::utils::GetAllowedEnumValuesString(storage_mode_mappings)); + +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +DEFINE_VALIDATED_string(storage_mode, "IN_MEMORY_TRANSACTIONAL", storage_mode_help_string.c_str(), { + if (const auto result = memgraph::utils::IsValidEnumValueString(value, storage_mode_mappings); result.HasError()) { + switch (result.GetError()) { + case memgraph::utils::ValidationError::EmptyValue: { + std::cout << "Storage mode cannot be empty." << std::endl; + break; + } + case memgraph::utils::ValidationError::InvalidValue: { + std::cout << "Invalid value for storage mode. Allowed values: " + << memgraph::utils::GetAllowedEnumValuesString(storage_mode_mappings) << std::endl; + break; + } + } + return false; + } + return true; +}); + +memgraph::storage::StorageMode memgraph::flags::ParseStorageMode() { + const auto storage_mode = + memgraph::utils::StringToEnum(FLAGS_storage_mode, storage_mode_mappings); + MG_ASSERT(storage_mode, "Invalid storage mode"); + return *storage_mode; +} diff --git a/src/flags/storage_mode.hpp b/src/flags/storage_mode.hpp new file mode 100644 index 000000000..0e756ae13 --- /dev/null +++ b/src/flags/storage_mode.hpp @@ -0,0 +1,19 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include "storage/v2/storage_mode.hpp" + +namespace memgraph::flags { + +memgraph::storage::StorageMode ParseStorageMode(); + +} // namespace memgraph::flags diff --git a/src/memgraph.cpp b/src/memgraph.cpp index b79a3c565..f28b463ab 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -291,7 +291,8 @@ int main(int argc, char **argv) { .name_id_mapper_directory = FLAGS_data_directory + "/rocksdb_name_id_mapper", .id_name_mapper_directory = FLAGS_data_directory + "/rocksdb_id_name_mapper", .durability_directory = FLAGS_data_directory + "/rocksdb_durability", - .wal_directory = FLAGS_data_directory + "/rocksdb_wal"}}; + .wal_directory = FLAGS_data_directory + "/rocksdb_wal"}, + .storage_mode = memgraph::flags::ParseStorageMode()}; if (FLAGS_storage_snapshot_interval_sec == 0) { if (FLAGS_storage_wal_enabled) { LOG_FATAL( diff --git a/src/storage/v2/config.hpp b/src/storage/v2/config.hpp index cd1590beb..53ed3a436 100644 --- a/src/storage/v2/config.hpp +++ b/src/storage/v2/config.hpp @@ -15,6 +15,7 @@ #include #include #include "storage/v2/isolation_level.hpp" +#include "storage/v2/storage_mode.hpp" #include "utils/exceptions.hpp" namespace memgraph::storage { @@ -79,6 +80,7 @@ struct Config { std::string name; bool force_on_disk{false}; + StorageMode storage_mode{StorageMode::IN_MEMORY_TRANSACTIONAL}; }; static inline void UpdatePaths(Config &config, const std::filesystem::path &storage_dir) { diff --git a/src/storage/v2/disk/rocksdb_storage.hpp b/src/storage/v2/disk/rocksdb_storage.hpp index b33f49feb..f36311aa3 100644 --- a/src/storage/v2/disk/rocksdb_storage.hpp +++ b/src/storage/v2/disk/rocksdb_storage.hpp @@ -78,7 +78,7 @@ class ComparatorWithU64TsImpl : public rocksdb::Comparator { }; struct DiskEdgeKey { - DiskEdgeKey(const std::string_view keyView) : key(keyView) {} + explicit DiskEdgeKey(const std::string_view keyView) : key(keyView) {} /// @tparam src_vertex_gid, dest_vertex_gid: Gid of the source and destination vertices /// @tparam edge_type_id: EdgeTypeId of the edge diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 7a4d579af..007cdf36f 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -22,13 +22,15 @@ namespace memgraph::storage { using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; -InMemoryStorage::InMemoryStorage(Config config) - : Storage(config, StorageMode::IN_MEMORY_TRANSACTIONAL), +InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) + : Storage(config, storage_mode), snapshot_directory_(config.durability.storage_directory / durability::kSnapshotDirectory), lock_file_path_(config.durability.storage_directory / durability::kLockFile), wal_directory_(config.durability.storage_directory / durability::kWalDirectory), uuid_(utils::GenerateUUID()), global_locker_(file_retainer_.AddLocker()) { + MG_ASSERT(storage_mode != StorageMode::ON_DISK_TRANSACTIONAL, + "Invalid storage mode sent to InMemoryStorage constructor!"); if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || config_.durability.snapshot_on_exit || config_.durability.recover_on_startup) { // Create the directory initially to crash the database in case of @@ -147,6 +149,8 @@ InMemoryStorage::InMemoryStorage(Config config) } } +InMemoryStorage::InMemoryStorage(Config config) : InMemoryStorage(config, StorageMode::IN_MEMORY_TRANSACTIONAL) {} + InMemoryStorage::~InMemoryStorage() { if (config_.gc.type == Config::Gc::Type::PERIODIC) { gc_runner_.Stop(); diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index cf9e9a062..301c8b139 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -51,6 +51,7 @@ class InMemoryStorage final : public Storage { /// @throw std::system_error /// @throw std::bad_alloc explicit InMemoryStorage(Config config = Config()); + InMemoryStorage(Config config, StorageMode storage_mode); InMemoryStorage(const InMemoryStorage &) = delete; InMemoryStorage(InMemoryStorage &&) = delete; diff --git a/tests/e2e/configuration/default_config.py b/tests/e2e/configuration/default_config.py index ba5e11c6e..d9c9bf769 100644 --- a/tests/e2e/configuration/default_config.py +++ b/tests/e2e/configuration/default_config.py @@ -156,6 +156,11 @@ startup_config_dict = { "100000", "Issue a 'fsync' call after this amount of transactions are written to the WAL file. Set to 1 for fully synchronous operation.", ), + "storage_mode": ( + "IN_MEMORY_TRANSACTIONAL", + "IN_MEMORY_TRANSACTIONAL", + "Default storage mode Memgraph uses. Allowed values: IN_MEMORY_TRANSACTIONAL, IN_MEMORY_ANALYTICAL, ON_DISK_TRANSACTIONAL", + ), "storage_wal_file_size_kib": ("20480", "20480", "Minimum file size of each WAL file."), "storage_delete_on_drop": ( "true",