tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add CREATE SNAPSHOT query (#182)

Browse Source
This commit is contained in:
antonio2368 2021-06-30 12:31:30 +02:00 committed by GitHub
parent 715162e205
commit 3b336e3e0b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 115 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/auth/models.cpp
View File

@ -41,8 +41,8 @@ std::string PermissionToString(Permission permission) {
return "DUMP";
case Permission::REPLICATION:
return "REPLICATION";
case Permission::LOCK_PATH:
return "LOCK_PATH";
case Permission::DURABILITY:
return "DURABILITY";
case Permission::READ_FILE:
return "READ_FILE";
case Permission::FREE_MEMORY:

4
src/auth/models.hpp
View File

@ -22,7 +22,7 @@ enum class Permission : uint64_t {
CONSTRAINT = 1U << 8U,
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
LOCK_PATH = 1U << 11U,
DURABILITY = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
@ -36,7 +36,7 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permissi
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::LOCK_PATH, Permission::READ_FILE, Permission::FREE_MEMORY,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY,
Permission::TRIGGER, Permission::CONFIG};
// Function that converts a permission to its string representation.

4
src/glue/auth.cpp
View File

@ -26,8 +26,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::DUMP;
case query::AuthQuery::Privilege::REPLICATION:
return auth::Permission::REPLICATION;
case query::AuthQuery::Privilege::LOCK_PATH:
return auth::Permission::LOCK_PATH;
case query::AuthQuery::Privilege::DURABILITY:
return auth::Permission::DURABILITY;
case query::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::AuthQuery::Privilege::FREE_MEMORY:

6
src/query/exceptions.hpp
View File

@ -181,4 +181,10 @@ class IsolationLevelModificationInMulticommandTxException : public QueryExceptio
IsolationLevelModificationInMulticommandTxException()
: QueryException("Isolation level cannot be modified in multicommand transactions.") {}
};
class CreateSnapshotInMulticommandTxException final : public QueryException {
public:
CreateSnapshotInMulticommandTxException()
: QueryException("Snapshot cannot be created in multicommand transactions.") {}
};
} // namespace query

12
src/query/frontend/ast/ast.lcp
View File

@ -2193,7 +2193,7 @@ cpp<#
(:serialize))
(lcp:define-enum privilege
(create delete match merge set remove index stats auth constraint
dump replication lock_path read_file free_memory trigger config)
dump replication durability read_file free_memory trigger config)
(:serialize))
#>cpp
AuthQuery() = default;
@ -2231,7 +2231,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP,
AuthQuery::Privilege::REPLICATION,
AuthQuery::Privilege::READ_FILE,
AuthQuery::Privilege::LOCK_PATH,
AuthQuery::Privilege::DURABILITY,
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
AuthQuery::Privilege::CONFIG};
cpp<#
@ -2456,4 +2456,12 @@ cpp<#
(:serialize (:slk))
(:clone))
(lcp:define-class create-snapshot-query (query) ()
(:public
#>cpp
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:pop-namespace) ;; namespace query

4
src/query/frontend/ast/ast_visitor.hpp
View File

@ -78,6 +78,7 @@ class LoadCsv;
class FreeMemoryQuery;
class TriggerQuery;
class IsolationLevelQuery;
class CreateSnapshotQuery;
using TreeCompositeVisitor = ::utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@ -111,6 +112,7 @@ class ExpressionVisitor
template <class TResult>
class QueryVisitor : public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery> {};
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery> {
};
} // namespace query

7
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@ -442,6 +442,11 @@ antlrcpp::Any CypherMainVisitor::visitIsolationLevelQuery(MemgraphCypher::Isolat
return isolation_level_query;
}
antlrcpp::Any CypherMainVisitor::visitCreateSnapshotQuery(MemgraphCypher::CreateSnapshotQueryContext *ctx) {
query_ = storage_->Create<CreateSnapshotQuery>();
return query_;
}
antlrcpp::Any CypherMainVisitor::visitCypherUnion(MemgraphCypher::CypherUnionContext *ctx) {
bool distinct = !ctx->ALL();
auto *cypher_union = storage_->Create<CypherUnion>(distinct);
@ -870,11 +875,11 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
if (ctx->CONSTRAINT()) return AuthQuery::Privilege::CONSTRAINT;
if (ctx->DUMP()) return AuthQuery::Privilege::DUMP;
if (ctx->REPLICATION()) return AuthQuery::Privilege::REPLICATION;
if (ctx->LOCK_PATH()) return AuthQuery::Privilege::LOCK_PATH;
if (ctx->READ_FILE()) return AuthQuery::Privilege::READ_FILE;
if (ctx->FREE_MEMORY()) return AuthQuery::Privilege::FREE_MEMORY;
if (ctx->TRIGGER()) return AuthQuery::Privilege::TRIGGER;
if (ctx->CONFIG()) return AuthQuery::Privilege::CONFIG;
if (ctx->DURABILITY()) return AuthQuery::Privilege::DURABILITY;
LOG_FATAL("Should not get here - unknown privilege!");
}

5
src/query/frontend/ast/cypher_main_visitor.hpp
View File

@ -243,6 +243,11 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) override;
/**
* @return CreateSnapshotQuery*
*/
antlrcpp::Any visitCreateSnapshotQuery(MemgraphCypher::CreateSnapshotQueryContext *ctx) override;
/**
* @return CypherUnion*
*/

5
src/query/frontend/opencypher/grammar/MemgraphCypher.g4
View File

@ -86,6 +86,7 @@ query : cypherQuery
| freeMemoryQuery
| triggerQuery
| isolationLevelQuery
| createSnapshotQuery
;
authQuery : createRole
@ -183,11 +184,11 @@ privilege : CREATE
| CONSTRAINT
| DUMP
| REPLICATION
| LOCK_PATH
| READ_FILE
| FREE_MEMORY
| TRIGGER
| CONFIG
| DURABILITY
;
privilegeList : privilege ( ',' privilege )* ;
@ -241,3 +242,5 @@ isolationLevel : SNAPSHOT ISOLATION | READ COMMITTED | READ UNCOMMITTED ;
isolationLevelScope : GLOBAL | SESSION | NEXT ;
isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ;
createSnapshotQuery : CREATE SNAPSHOT ;

2
src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4
View File

@ -30,6 +30,7 @@ DENY : D E N Y ;
DIRECTORY : D I R E C T O R Y ;
DROP : D R O P ;
DUMP : D U M P ;
DURABILITY : D U R A B I L I T Y ;
EXECUTE : E X E C U T E ;
FOR : F O R ;
FREE : F R E E ;
@ -45,7 +46,6 @@ ISOLATION : I S O L A T I O N ;
LEVEL : L E V E L ;
LOAD : L O A D ;
LOCK : L O C K ;
LOCK_PATH : L O C K UNDERSCORE P A T H ;
MAIN : M A I N ;
MODE : M O D E ;
NEXT : N E X T ;

4
src/query/frontend/semantic/required_privileges.cpp
View File

@ -49,7 +49,7 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(DumpQuery &dump_query) override { AddPrivilege(AuthQuery::Privilege::DUMP); }
void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::LOCK_PATH); }
void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); }
void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); }
@ -59,6 +59,8 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(IsolationLevelQuery &isolation_level_query) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
void Visit(CreateSnapshotQuery &create_snapshot_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); }
bool PreVisit(Create & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::CREATE);
return false;

26
src/query/interpreter.cpp
View File

@ -6,6 +6,7 @@
#include "glue/communication.hpp"
#include "query/constants.hpp"
#include "query/context.hpp"
#include "query/cypher_query_interpreter.hpp"
#include "query/db_accessor.hpp"
#include "query/dump.hpp"
#include "query/exceptions.hpp"
@ -1207,6 +1208,28 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in
RWType::NONE};
}
PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
InterpreterContext *interpreter_context) {
if (in_explicit_transaction) {
throw CreateSnapshotInMulticommandTxException();
}
return PreparedQuery{
{},
std::move(parsed_query.required_privileges),
[interpreter_context](AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
if (auto maybe_error = interpreter_context->db->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::Storage::CreateSnapshotError::DisabledForReplica:
throw utils::BasicException(
"Failed to create a snapshot. Replica instances are not allowed to create them.");
}
}
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
}
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MemoryResource *execution_memory) {
@ -1552,6 +1575,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<IsolationLevelQuery>(parsed_query.query)) {
prepared_query =
PrepareIsolationLevelQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_, this);
} else if (utils::Downcast<CreateSnapshotQuery>(parsed_query.query)) {
prepared_query =
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
} else {
LOG_FATAL("Should not get here -- unknown query type!");
}

26
src/storage/v2/storage.cpp
View File

@ -361,7 +361,15 @@ Storage::Storage(Config config)
}
}
if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED) {
snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval, [this] { this->CreateSnapshot(); });
snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval, [this] {
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case CreateSnapshotError::DisabledForReplica:
spdlog::warn("Snapshots are disabled for replicas!");
break;
}
}
});
}
if (config_.gc.type == Config::Gc::Type::PERIODIC) {
gc_runner_.Run("Storage GC", config_.gc.interval, [this] { this->CollectGarbage<false>(); });
@ -391,7 +399,13 @@ Storage::~Storage() {
snapshot_runner_.Stop();
}
if (config_.durability.snapshot_on_exit) {
CreateSnapshot();
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case CreateSnapshotError::DisabledForReplica:
spdlog::warn("Snapshots are disabled for replicas!");
break;
}
}
}
}
@ -1727,12 +1741,13 @@ void Storage::AppendToWal(durability::StorageGlobalOperation operation, LabelId
FinalizeWalFile();
}
void Storage::CreateSnapshot() {
utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot() {
if (replication_role_.load() != ReplicationRole::MAIN) {
spdlog::warn("Snapshots are disabled for replicas!");
return;
return CreateSnapshotError::DisabledForReplica;
}
std::lock_guard snapshot_guard(snapshot_lock_);
// Take master RW lock (for reading).
std::shared_lock<utils::RWLock> storage_guard(main_lock_);
@ -1747,6 +1762,7 @@ void Storage::CreateSnapshot() {
// Finalize snapshot transaction.
commit_log_->MarkFinished(transaction.start_timestamp);
return {};
}
bool Storage::LockPath() {

7
src/storage/v2/storage.hpp
View File

@ -428,6 +428,10 @@ class Storage final {
void SetIsolationLevel(IsolationLevel isolation_level);
enum class CreateSnapshotError : uint8_t { DisabledForReplica };
utils::BasicResult<CreateSnapshotError> CreateSnapshot();
private:
Transaction CreateTransaction(IsolationLevel isolation_level);
@ -452,8 +456,6 @@ class Storage final {
void AppendToWal(durability::StorageGlobalOperation operation, LabelId label, const std::set<PropertyId> &properties,
uint64_t final_commit_timestamp);
void CreateSnapshot();
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});
// Main storage lock.
@ -518,6 +520,7 @@ class Storage final {
utils::OutputFile lock_file_handle_;
utils::Scheduler snapshot_runner_;
utils::SpinLock snapshot_lock_;
// UUID used to distinguish snapshots and to link snapshots to WALs
std::string uuid_;

9
tests/unit/cypher_main_visitor.cpp
View File

@ -2057,8 +2057,8 @@ TEST_P(CypherMainVisitorTest, GrantPrivilege) {
{AuthQuery::Privilege::DUMP});
check_auth_query(&ast_generator, "GRANT REPLICATION TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
{AuthQuery::Privilege::REPLICATION});
check_auth_query(&ast_generator, "GRANT LOCK_PATH TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
{AuthQuery::Privilege::LOCK_PATH});
check_auth_query(&ast_generator, "GRANT DURABILITY TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
{AuthQuery::Privilege::DURABILITY});
check_auth_query(&ast_generator, "GRANT READ_FILE TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
{AuthQuery::Privilege::READ_FILE});
check_auth_query(&ast_generator, "GRANT FREE_MEMORY TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
@ -3198,4 +3198,9 @@ TEST_P(CypherMainVisitorTest, SetIsolationLevelQuery) {
}
}
}
TEST_P(CypherMainVisitorTest, CreateSnapshotQuery) {
auto &ast_generator = *GetParam();
ASSERT_TRUE(dynamic_cast<CreateSnapshotQuery *>(ast_generator.ParseQuery("CREATE SNAPSHOT")));
}
} // namespace

8
tests/unit/query_required_privileges.cpp
View File

@ -1,6 +1,7 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/frontend/semantic/required_privileges.hpp"
#include "storage/v2/id_types.hpp"
@ -142,7 +143,7 @@ TEST_F(TestPrivilegeExtractor, ReadFile) {
TEST_F(TestPrivilegeExtractor, LockPathQuery) {
auto *query = storage.Create<LockPathQuery>();
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::LOCK_PATH));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::DURABILITY));
}
TEST_F(TestPrivilegeExtractor, FreeMemoryQuery) {
@ -159,3 +160,8 @@ TEST_F(TestPrivilegeExtractor, SetIsolationLevelQuery) {
auto *query = storage.Create<IsolationLevelQuery>();
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::CONFIG));
}
TEST_F(TestPrivilegeExtractor, CreateSnapshotQuery) {
auto *query = storage.Create<CreateSnapshotQuery>();
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::DURABILITY));
}