tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add data directory status and (un)lock query (#933)

Browse Source
This commit is contained in:
andrejtonev 2023-05-16 18:36:04 +02:00 committed by GitHub
parent 7ddce539fa
commit 802f8aceda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 172 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/query/frontend/ast/ast.hpp
View File

@ -2898,7 +2898,7 @@ class LockPathQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class Action { LOCK_PATH, UNLOCK_PATH };
enum class Action { LOCK_PATH, UNLOCK_PATH, STATUS };
LockPathQuery() = default;

4
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@ -328,7 +328,9 @@ antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasC
antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) {
auto *lock_query = storage_->Create<LockPathQuery>();
if (ctx->LOCK()) {
if (ctx->STATUS()) {
lock_query->action_ = LockPathQuery::Action::STATUS;
} else if (ctx->LOCK()) {
lock_query->action_ = LockPathQuery::Action::LOCK_PATH;
} else if (ctx->UNLOCK()) {
lock_query->action_ = LockPathQuery::Action::UNLOCK_PATH;

3
src/query/frontend/opencypher/grammar/MemgraphCypher.g4
View File

@ -91,6 +91,7 @@ memgraphCypherKeyword : cypherKeyword
| SNAPSHOT
| START
| STATS
| STATUS
| STORAGE
| STREAM
| STREAMS
@ -335,7 +336,7 @@ dropReplica : DROP REPLICA replicaName ;
showReplicas : SHOW REPLICAS ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY | DATA DIRECTORY LOCK STATUS;
freeMemoryQuery : FREE MEMORY ;

1
src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4
View File

@ -107,6 +107,7 @@ SNAPSHOT : S N A P S H O T ;
START : S T A R T ;
STATISTICS : S T A T I S T I C S ;
STATS : S T A T S ;
STATUS : S T A T U S ;
STOP : S T O P ;
STORAGE : S T O R A G E;
STORAGE_MODE : S T O R A G E UNDERSCORE MODE;

5
src/query/frontend/stripped_lexer_constants.hpp
View File

@ -145,6 +145,7 @@ const trie::Trie kKeywords = {"union",
"drop",
"show",
"stats",
"status",
"unique",
"explain",
"profile",
@ -212,6 +213,10 @@ const trie::Trie kKeywords = {"union",
"off",
"in_memory_transactional",
"in_memory_analytical",
"data",
"directory",
"lock",
"unlock"
"build"};
// Unicode codepoints that are allowed at the start of the unescaped name.

62
src/query/interpreter.cpp
View File

@ -1780,25 +1780,49 @@ PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_tr
auto *lock_path_query = utils::Downcast<LockPathQuery>(parsed_query.query);
return PreparedQuery{{},
std::move(parsed_query.required_privileges),
[interpreter_context, action = lock_path_query->action_](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
switch (action) {
case LockPathQuery::Action::LOCK_PATH:
if (!interpreter_context->db->LockPath()) {
throw QueryRuntimeException("Failed to lock the data directory");
}
break;
case LockPathQuery::Action::UNLOCK_PATH:
if (!interpreter_context->db->UnlockPath()) {
throw QueryRuntimeException("Failed to unlock the data directory");
}
break;
}
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
return PreparedQuery{
{"STATUS"},
std::move(parsed_query.required_privileges),
[interpreter_context, action = lock_path_query->action_](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
std::vector<std::vector<TypedValue>> status;
std::string res;
switch (action) {
case LockPathQuery::Action::LOCK_PATH: {
const auto lock_success = interpreter_context->db->LockPath();
if (lock_success.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to lock the data directory");
}
res = lock_success.GetValue() ? "Data directory is now locked." : "Data directory is already locked.";
break;
}
case LockPathQuery::Action::UNLOCK_PATH: {
const auto unlock_success = interpreter_context->db->UnlockPath();
if (unlock_success.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to unlock the data directory");
}
res = unlock_success.GetValue() ? "Data directory is now unlocked." : "Data directory is already unlocked.";
break;
}
case LockPathQuery::Action::STATUS: {
const auto locked_status = interpreter_context->db->IsPathLocked();
if (locked_status.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to access the data directory");
}
res = locked_status.GetValue() ? "Data directory is locked." : "Data directory is unlocked.";
break;
}
}
status.emplace_back(std::vector<TypedValue>{TypedValue(res)});
auto pull_plan = std::make_shared<PullPlanVector>(std::move(status));
if (pull_plan->Pull(stream, n)) {
return QueryHandlerResult::COMMIT;
}
return std::nullopt;
},
RWType::NONE};
}
PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, bool in_explicit_transaction,

15
src/storage/v2/replication/replication_client.cpp
View File

@ -399,7 +399,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
// we cannot know if the difference is only in the current WAL or we need
// to send the snapshot.
if (latest_snapshot) {
locker_acc.AddPath(latest_snapshot->path);
const auto lock_success = locker_acc.AddPath(latest_snapshot->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_steps.emplace_back(std::in_place_type_t<RecoverySnapshot>{}, std::move(latest_snapshot->path));
}
// if there are no finalized WAL files, snapshot left the current WAL
@ -446,7 +447,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
// We need to lock these files and add them to the chain
for (auto result_wal_it = wal_files->begin() + distance_from_first; result_wal_it != wal_files->end();
++result_wal_it) {
locker_acc.AddPath(result_wal_it->path);
const auto lock_success = locker_acc.AddPath(result_wal_it->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
wal_chain.push_back(std::move(result_wal_it->path));
}
@ -464,7 +466,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
MG_ASSERT(latest_snapshot, "Invalid durability state, missing snapshot");
// We didn't manage to find a WAL chain, we need to send the latest snapshot
// with its WALs
locker_acc.AddPath(latest_snapshot->path);
const auto lock_success = locker_acc.AddPath(latest_snapshot->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_steps.emplace_back(std::in_place_type_t<RecoverySnapshot>{}, std::move(latest_snapshot->path));
std::vector<std::filesystem::path> recovery_wal_files;
@ -483,13 +486,15 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
}
for (; wal_it != wal_files->end(); ++wal_it) {
locker_acc.AddPath(wal_it->path);
const auto lock_success = locker_acc.AddPath(wal_it->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_wal_files.push_back(std::move(wal_it->path));
}
// We only have a WAL before the snapshot
if (recovery_wal_files.empty()) {
locker_acc.AddPath(wal_files->back().path);
const auto lock_success = locker_acc.AddPath(wal_files->back().path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_wal_files.push_back(std::move(wal_files->back().path));
}

15
src/storage/v2/storage.cpp
View File

@ -1980,16 +1980,23 @@ utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot(std::op
return CreateSnapshotError::ReachedMaxNumTries;
}
bool Storage::LockPath() {
utils::FileRetainer::FileLockerAccessor::ret_type Storage::IsPathLocked() {
auto locker_accessor = global_locker_.Access();
return locker_accessor.IsPathLocked(config_.durability.storage_directory);
}
utils::FileRetainer::FileLockerAccessor::ret_type Storage::LockPath() {
auto locker_accessor = global_locker_.Access();
return locker_accessor.AddPath(config_.durability.storage_directory);
}
bool Storage::UnlockPath() {
utils::FileRetainer::FileLockerAccessor::ret_type Storage::UnlockPath() {
{
auto locker_accessor = global_locker_.Access();
if (!locker_accessor.RemovePath(config_.durability.storage_directory)) {
return false;
const auto ret = locker_accessor.RemovePath(config_.durability.storage_directory);
if (ret.HasError() || !ret.GetValue()) {
// Exit without cleaning the queue
return ret;
}
}

6
src/storage/v2/storage.hpp
View File

@ -39,6 +39,7 @@
#include "storage/v2/vertex_accessor.hpp"
#include "utils/file_locker.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/result.hpp"
#include "utils/rw_lock.hpp"
#include "utils/scheduler.hpp"
#include "utils/skip_list.hpp"
@ -467,8 +468,9 @@ class Storage final {
StorageInfo GetInfo() const;
bool LockPath();
bool UnlockPath();
utils::FileRetainer::FileLockerAccessor::ret_type IsPathLocked();
utils::FileRetainer::FileLockerAccessor::ret_type LockPath();
utils::FileRetainer::FileLockerAccessor::ret_type UnlockPath();
bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config = {});

35
src/utils/file_locker.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -80,13 +80,14 @@ void FileRetainer::CleanQueue() {
}
////// LockerEntry //////
void FileRetainer::LockerEntry::LockPath(const std::filesystem::path &path) {
bool FileRetainer::LockerEntry::LockPath(const std::filesystem::path &path) {
auto absolute_path = std::filesystem::absolute(path);
if (std::filesystem::is_directory(absolute_path)) {
directories_.emplace(std::move(absolute_path));
return;
const auto [itr, success] = directories_.emplace(std::move(absolute_path));
return success;
}
files_.emplace(std::move(absolute_path));
const auto [itr, success] = files_.emplace(std::move(absolute_path));
return success;
}
bool FileRetainer::LockerEntry::RemovePath(const std::filesystem::path &path) {
@ -140,13 +141,27 @@ FileRetainer::FileLockerAccessor::FileLockerAccessor(FileRetainer *retainer, siz
file_retainer_->active_accessors_.fetch_add(1);
}
bool FileRetainer::FileLockerAccessor::AddPath(const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) return false;
file_retainer_->lockers_.WithLock([&](auto &lockers) { lockers[locker_id_].LockPath(path); });
return true;
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::IsPathLocked(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->FileLocked(std::filesystem::absolute(path));
}
bool FileRetainer::FileLockerAccessor::RemovePath(const std::filesystem::path &path) {
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::AddPath(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->lockers_.WithLock([&](auto &lockers) { return lockers[locker_id_].LockPath(path); });
}
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::RemovePath(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->lockers_.WithLock([&](auto &lockers) { return lockers[locker_id_].RemovePath(path); });
}

20
src/utils/file_locker.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -18,6 +18,7 @@
#include <unordered_map>
#include "utils/file.hpp"
#include "utils/result.hpp"
#include "utils/rw_lock.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
@ -114,15 +115,26 @@ class FileRetainer {
struct FileLockerAccessor {
friend FileLocker;
enum class Error : uint8_t {
NonexistentPath = 0,
};
using ret_type = utils::BasicResult<FileRetainer::FileLockerAccessor::Error, bool>;
/**
* Checks if a single path is in the current locker.
*/
ret_type IsPathLocked(const std::filesystem::path &path);
/**
* Add a single path to the current locker.
*/
bool AddPath(const std::filesystem::path &path);
ret_type AddPath(const std::filesystem::path &path);
/**
* Remove a single path form the current locker.
*/
bool RemovePath(const std::filesystem::path &path);
ret_type RemovePath(const std::filesystem::path &path);
FileLockerAccessor(const FileLockerAccessor &) = delete;
FileLockerAccessor(FileLockerAccessor &&) = default;
@ -182,7 +194,7 @@ class FileRetainer {
class LockerEntry {
public:
void LockPath(const std::filesystem::path &path);
bool LockPath(const std::filesystem::path &path);
bool RemovePath(const std::filesystem::path &path);
[[nodiscard]] bool LocksFile(const std::filesystem::path &path) const;

33
tests/unit/cypher_main_visitor.cpp
View File

@ -3368,10 +3368,43 @@ TEST_P(CypherMainVisitorTest, TestLockPathQuery) {
ASSERT_TRUE(parsed_query);
EXPECT_EQ(parsed_query->action_, action);
}
{
const std::string query = fmt::format("{} DATA DIRECTORY LOCK STATUS", command);
ASSERT_THROW(ast_generator.ParseQuery(query), SyntaxException);
}
{
const std::string query = fmt::format("{} DATA DIRECTORY STATUS", command);
ASSERT_THROW(ast_generator.ParseQuery(query), SyntaxException);
}
};
test_lock_path_query("LOCK", LockPathQuery::Action::LOCK_PATH);
test_lock_path_query("UNLOCK", LockPathQuery::Action::UNLOCK_PATH);
// Status test
{
const std::string query = "DATA DIRECTORY LOCK";
ASSERT_THROW(ast_generator.ParseQuery(query), SyntaxException);
}
{
const std::string query = "DATA LOCK STATUS";
ASSERT_THROW(ast_generator.ParseQuery(query), SyntaxException);
}
{
const std::string query = "DIRECTORY LOCK STATUS";
ASSERT_THROW(ast_generator.ParseQuery(query), SyntaxException);
}
{
const std::string query = "DATA DIRECTORY LOCK STATUS";
auto *parsed_query = dynamic_cast<LockPathQuery *>(ast_generator.ParseQuery(query));
ASSERT_TRUE(parsed_query);
EXPECT_EQ(parsed_query->action_, LockPathQuery::Action::STATUS);
}
}
TEST_P(CypherMainVisitorTest, TestLoadCsvClause) {

27
tests/unit/utils_file_locker.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -89,7 +89,8 @@ TEST_P(FileLockerParameterizedTest, DeleteWhileInLocker) {
auto locker = file_retainer.AddLocker();
{
auto acc = locker.Access();
acc.AddPath(lock_absolute ? file_absolute : file);
const auto lock_success = acc.AddPath(lock_absolute ? file_absolute : file);
ASSERT_FALSE(lock_success.HasError());
}
file_retainer.DeleteFile(delete_absolute ? file_absolute : file);
@ -133,7 +134,9 @@ TEST_P(FileLockerParameterizedTest, DirectoryLock) {
auto locker = file_retainer.AddLocker();
{
auto acc = locker.Access();
acc.AddPath(lock_absolute ? std::filesystem::absolute(directory_to_lock) : directory_to_lock);
const auto lock_success =
acc.AddPath(lock_absolute ? std::filesystem::absolute(directory_to_lock) : directory_to_lock);
ASSERT_FALSE(lock_success.HasError());
}
file_retainer.DeleteFile(delete_absolute ? file_absolute : file);
@ -165,7 +168,8 @@ TEST_P(FileLockerParameterizedTest, RemovePath) {
auto locker = file_retainer.AddLocker();
{
auto acc = locker.Access();
acc.AddPath(lock_absolute ? file_absolute : file);
const auto lock_success = acc.AddPath(lock_absolute ? file_absolute : file);
ASSERT_FALSE(lock_success.HasError());
}
file_retainer.DeleteFile(delete_absolute ? file_absolute : file);
@ -205,8 +209,10 @@ TEST_F(FileLockerTest, MultipleLockers) {
auto locker = file_retainer.AddLocker();
{
auto acc = locker.Access();
acc.AddPath(file1);
acc.AddPath(common_file);
const auto lock_success1 = acc.AddPath(file1);
ASSERT_FALSE(lock_success1.HasError());
const auto lock_success2 = acc.AddPath(common_file);
ASSERT_FALSE(lock_success2.HasError());
}
std::this_thread::sleep_for(200ms);
});
@ -215,8 +221,10 @@ TEST_F(FileLockerTest, MultipleLockers) {
auto locker = file_retainer.AddLocker();
{
auto acc = locker.Access();
acc.AddPath(file2);
acc.AddPath(common_file);
const auto lock_success1 = acc.AddPath(file2);
ASSERT_FALSE(lock_success1.HasError());
const auto lock_success2 = acc.AddPath(common_file);
ASSERT_FALSE(lock_success2.HasError());
}
std::this_thread::sleep_for(200ms);
});
@ -275,7 +283,8 @@ TEST_F(FileLockerTest, MultipleLockersAndDeleters) {
auto acc = locker.Access();
for (auto i = 0; i < file_access_num; ++i) {
auto file = random_file();
if (acc.AddPath(file)) {
const auto res = acc.AddPath(file);
if (!res.HasError()) {
ASSERT_TRUE(std::filesystem::exists(file));
locked_files.emplace_back(std::move(file));
} else {