From 6182312e3d2d9bd7c06939f5804b0ba876092561 Mon Sep 17 00:00:00 2001 From: Matej Ferencevic Date: Thu, 25 Apr 2019 11:10:29 +0200 Subject: [PATCH] Remove TX info from HA snapshot Reviewers: msantl, ipaljak Reviewed By: msantl Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D1984 --- src/durability/single_node_ha/paths.cpp | 43 +++++-------------- src/durability/single_node_ha/paths.hpp | 16 +++---- src/durability/single_node_ha/recovery.cpp | 19 -------- src/durability/single_node_ha/recovery.hpp | 8 ---- src/durability/single_node_ha/snapshooter.cpp | 14 +----- src/durability/single_node_ha/version.hpp | 20 +++------ src/raft/raft_server.cpp | 4 +- src/transactions/single_node_ha/engine.cpp | 5 --- src/transactions/single_node_ha/engine.hpp | 1 - 9 files changed, 25 insertions(+), 105 deletions(-) diff --git a/src/durability/single_node_ha/paths.cpp b/src/durability/single_node_ha/paths.cpp index 34bdfaa99..fb2b6f4a8 100644 --- a/src/durability/single_node_ha/paths.cpp +++ b/src/durability/single_node_ha/paths.cpp @@ -1,12 +1,5 @@ #include "durability/single_node_ha/paths.hpp" -#include -#include -#include - -#include "glog/logging.h" - -#include "transactions/type.hpp" #include "utils/string.hpp" #include "utils/timestamp.hpp" @@ -14,36 +7,20 @@ namespace durability { namespace fs = std::filesystem; -std::string GetSnapshotFilename(tx::TransactionId tx_id) { - std::string date_str = - utils::Timestamp(utils::Timestamp::Now()) - .ToString("{:04d}_{:02d}_{:02d}__{:02d}_{:02d}_{:02d}_{:05d}"); - return date_str + "_tx_" + std::to_string(tx_id); +// This is the prefix used for WAL and Snapshot filenames. It is a timestamp +// format that equals to: YYYYmmddHHMMSSffffff +const std::string kTimestampFormat = + "{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}{:06d}"; + +std::string GetSnapshotFilename(uint64_t last_included_term, + uint64_t last_included_index) { + std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat); + return date_str + "_term_" + std::to_string(last_included_term) + "_index_" + + std::to_string(last_included_index); } fs::path MakeSnapshotPath(const fs::path &durability_dir, const std::string &snapshot_filename) { return durability_dir / kSnapshotDir / snapshot_filename; } - -std::optional TransactionIdFromSnapshotFilename( - const std::string &name) { - auto nullopt = std::nullopt; - auto file_name_split = utils::RSplit(name, "_tx_", 1); - if (file_name_split.size() != 2) { - LOG(WARNING) << "Unable to parse snapshot file name: " << name; - return nullopt; - } - try { - return std::stoll(file_name_split[1]); - } catch (std::invalid_argument &) { - LOG(WARNING) << "Unable to parse snapshot file name tx ID: " - << file_name_split[1]; - return nullopt; - } catch (std::out_of_range &) { - LOG(WARNING) << "Unable to parse snapshot file name tx ID: " - << file_name_split[1]; - return nullopt; - } -} } // namespace durability diff --git a/src/durability/single_node_ha/paths.hpp b/src/durability/single_node_ha/paths.hpp index f3c1f9ae2..ec2d36f19 100644 --- a/src/durability/single_node_ha/paths.hpp +++ b/src/durability/single_node_ha/paths.hpp @@ -1,26 +1,20 @@ #pragma once #include -#include - -#include "transactions/type.hpp" +#include namespace durability { const std::string kSnapshotDir = "snapshots"; const std::string kBackupDir = ".backup"; /// Generates a filename for a DB snapshot in the given folder in a well-defined -/// sortable format with transaction from which the snapshot is created appended -/// to the file name. -std::string GetSnapshotFilename(tx::TransactionId tx_id); +/// sortable format with last included term and last included index from which +/// the snapshot is created appended to the file name. +std::string GetSnapshotFilename(uint64_t last_included_term, + uint64_t last_included_index); /// Generates a full path for a DB snapshot. std::filesystem::path MakeSnapshotPath( const std::filesystem::path &durability_dir, const std::string &snapshot_filename); - -/// Returns the transaction id contained in the file name. If the filename is -/// not a parseable snapshot file name, nullopt is returned. -std::optional TransactionIdFromSnapshotFilename( - const std::string &name); } // namespace durability diff --git a/src/durability/single_node_ha/recovery.cpp b/src/durability/single_node_ha/recovery.cpp index 1622b3da5..fb4329a7e 100644 --- a/src/durability/single_node_ha/recovery.cpp +++ b/src/durability/single_node_ha/recovery.cpp @@ -64,15 +64,6 @@ bool RecoverSnapshot(const fs::path &snapshot_file, database::GraphDb *db, RETURN_IF_NOT(decoder.ReadValue(&dv, Value::Type::Int) && dv.ValueInt() == durability::kVersion); - RETURN_IF_NOT(decoder.ReadValue(&dv, Value::Type::Int)); - recovery_data->snapshooter_tx_id = dv.ValueInt(); - // Transaction snapshot of the transaction that created the snapshot. - RETURN_IF_NOT(decoder.ReadValue(&dv, Value::Type::List)); - for (const auto &value : dv.ValueList()) { - RETURN_IF_NOT(value.IsInt()); - recovery_data->snapshooter_tx_snapshot.emplace_back(value.ValueInt()); - } - // A list of label+property indexes. RETURN_IF_NOT(decoder.ReadValue(&dv, Value::Type::List)); auto index_value = dv.ValueList(); @@ -131,16 +122,6 @@ bool RecoverSnapshot(const fs::path &snapshot_file, database::GraphDb *db, return false; } - // Ensure that the next transaction ID in the recovered DB will be greater - // than the latest one we have recovered. Do this to make sure that - // subsequently created snapshots files will have transactional info - // that does not interfere with that found in previous snapshots. - tx::TransactionId max_id = recovery_data->snapshooter_tx_id; - auto &snap = recovery_data->snapshooter_tx_snapshot; - if (!snap.empty()) { - max_id = std::max(max_id, *std::max_element(snap.begin(), snap.end())); - } - dba.db()->tx_engine().EnsureNextIdGreater(max_id); dba.Commit(); return true; } diff --git a/src/durability/single_node_ha/recovery.hpp b/src/durability/single_node_ha/recovery.hpp index 5e0f3cb13..8177b8a42 100644 --- a/src/durability/single_node_ha/recovery.hpp +++ b/src/durability/single_node_ha/recovery.hpp @@ -25,17 +25,9 @@ struct IndexRecoveryData { /// Data structure for exchanging info between main recovery function and /// snapshot recovery functions. struct RecoveryData { - tx::TransactionId snapshooter_tx_id{0}; - std::vector snapshooter_tx_snapshot; // A collection into which the indexes should be added so they // can be rebuilt at the end of the recovery transaction. std::vector indexes; - - void Clear() { - snapshooter_tx_id = 0; - snapshooter_tx_snapshot.clear(); - indexes.clear(); - } }; /// Reads snapshot metadata from the end of the file without messing up the diff --git a/src/durability/single_node_ha/snapshooter.cpp b/src/durability/single_node_ha/snapshooter.cpp index ae592cd5c..ead384694 100644 --- a/src/durability/single_node_ha/snapshooter.cpp +++ b/src/durability/single_node_ha/snapshooter.cpp @@ -17,7 +17,7 @@ namespace fs = std::filesystem; namespace durability { // Snapshot layout is described in durability/version.hpp -static_assert(durability::kVersion == 7, +static_assert(durability::kVersion == 8, "Wrong snapshot version, please update!"); namespace { @@ -32,18 +32,6 @@ bool Encode(const fs::path &snapshot_file, database::GraphDb &db, durability::kSnapshotMagic.size()); encoder.WriteInt(durability::kVersion); - // Write the ID of the transaction doing the snapshot. - encoder.WriteInt(dba.transaction_id()); - - // Write the transaction snapshot into the snapshot. It's used when - // recovering from the combination of snapshot and write-ahead-log. - { - std::vector tx_snapshot; - for (int64_t tx : dba.transaction().snapshot()) - tx_snapshot.emplace_back(tx); - encoder.WriteList(tx_snapshot); - } - // Write label+property indexes as list ["label", "property", ...] { std::vector index_vec; diff --git a/src/durability/single_node_ha/version.hpp b/src/durability/single_node_ha/version.hpp index e3a90decc..fc437e3f3 100644 --- a/src/durability/single_node_ha/version.hpp +++ b/src/durability/single_node_ha/version.hpp @@ -11,30 +11,24 @@ namespace durability { -constexpr std::array kSnapshotMagic{{'M', 'G', 's', 'n'}}; -constexpr std::array kWalMagic{{'M', 'G', 'w', 'l'}}; +constexpr std::array kSnapshotMagic{{'M', 'G', 'H', 'A', 's', 'n'}}; // The current default version of snapshot and WAL encoding / decoding. -constexpr int64_t kVersion{7}; +constexpr int64_t kVersion{8}; -// Snapshot format (version 7): +// Snapshot format (version 8): // 1) Magic number + snapshot version // -// The following two entries are required when recovering from snapshot combined -// with WAL to determine record visibility. -// 2) Transactional ID of the snapshooter -// 3) Transactional snapshot of the snapshooter +// 2) A list of label+property indices. // -// 4) A list of label+property indices. -// -// 5) Bolt encoded nodes. Each node is written in the following format: +// 3) Bolt encoded nodes. Each node is written in the following format: // * gid, labels, properties -// 6) Bolt encoded edges. Each edge is written in the following format: +// 4) Bolt encoded edges. Each edge is written in the following format: // * gid // * from, to // * edge_type // * properties // -// 7) Snapshot summary (number of nodes, number of edges, hash) +// 5) Snapshot summary (number of nodes, number of edges, hash) } // namespace durability diff --git a/src/raft/raft_server.cpp b/src/raft/raft_server.cpp index 53f71d30f..bee2897f7 100644 --- a/src/raft/raft_server.cpp +++ b/src/raft/raft_server.cpp @@ -971,8 +971,8 @@ void RaftServer::SnapshotThread() { auto dba = db_->Access(); last_included_term = GetLogEntry(last_applied_).term; last_included_index = last_applied_; - snapshot_filename = - durability::GetSnapshotFilename(dba.transaction_id()); + snapshot_filename = durability::GetSnapshotFilename( + last_included_term, last_included_index); lock.unlock(); VLOG(40) << "[LogCompaction] Creating snapshot."; diff --git a/src/transactions/single_node_ha/engine.cpp b/src/transactions/single_node_ha/engine.cpp index 5f3e37d04..681f59c01 100644 --- a/src/transactions/single_node_ha/engine.cpp +++ b/src/transactions/single_node_ha/engine.cpp @@ -190,11 +190,6 @@ Transaction *Engine::RunningTransaction(TransactionId tx_id) { return found->second.get(); } -void Engine::EnsureNextIdGreater(TransactionId tx_id) { - std::lock_guard guard(lock_); - counter_ = std::max(tx_id, counter_); -} - void Engine::Reset() { Snapshot wait_for_txs; { diff --git a/src/transactions/single_node_ha/engine.hpp b/src/transactions/single_node_ha/engine.hpp index b8b293636..8c8647f38 100644 --- a/src/transactions/single_node_ha/engine.hpp +++ b/src/transactions/single_node_ha/engine.hpp @@ -49,7 +49,6 @@ class Engine final { TransactionId LocalOldestActive() const; void LocalForEachActiveTransaction(std::function f); Transaction *RunningTransaction(TransactionId tx_id); - void EnsureNextIdGreater(TransactionId tx_id); void GarbageCollectCommitLog(TransactionId tx_id); auto &local_lock_graph() { return local_lock_graph_; }