diff --git a/src/storage/v2/durability.cpp b/src/storage/v2/durability.cpp index 4d92db660..f64110efa 100644 --- a/src/storage/v2/durability.cpp +++ b/src/storage/v2/durability.cpp @@ -123,6 +123,8 @@ void Encoder::SetPosition(uint64_t position) { file_.SetPosition(utils::OutputFile::Position::SET, position); } +void Encoder::Sync() { file_.Sync(); } + void Encoder::Finalize() { file_.Sync(); file_.Close(); @@ -303,7 +305,23 @@ std::optional Decoder::ReadPropertyValue() { case Marker::SECTION_METADATA: case Marker::SECTION_INDICES: case Marker::SECTION_CONSTRAINTS: + case Marker::SECTION_DELTA: case Marker::SECTION_OFFSETS: + case Marker::DELTA_VERTEX_CREATE: + case Marker::DELTA_VERTEX_DELETE: + case Marker::DELTA_VERTEX_ADD_LABEL: + case Marker::DELTA_VERTEX_REMOVE_LABEL: + case Marker::DELTA_VERTEX_SET_PROPERTY: + case Marker::DELTA_EDGE_CREATE: + case Marker::DELTA_EDGE_DELETE: + case Marker::DELTA_EDGE_SET_PROPERTY: + case Marker::DELTA_TRANSACTION_END: + case Marker::DELTA_LABEL_INDEX_CREATE: + case Marker::DELTA_LABEL_INDEX_DROP: + case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: + case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: + case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: + case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: case Marker::VALUE_FALSE: case Marker::VALUE_TRUE: return std::nullopt; @@ -380,7 +398,23 @@ bool Decoder::SkipPropertyValue() { case Marker::SECTION_METADATA: case Marker::SECTION_INDICES: case Marker::SECTION_CONSTRAINTS: + case Marker::SECTION_DELTA: case Marker::SECTION_OFFSETS: + case Marker::DELTA_VERTEX_CREATE: + case Marker::DELTA_VERTEX_DELETE: + case Marker::DELTA_VERTEX_ADD_LABEL: + case Marker::DELTA_VERTEX_REMOVE_LABEL: + case Marker::DELTA_VERTEX_SET_PROPERTY: + case Marker::DELTA_EDGE_CREATE: + case Marker::DELTA_EDGE_DELETE: + case Marker::DELTA_EDGE_SET_PROPERTY: + case Marker::DELTA_TRANSACTION_END: + case Marker::DELTA_LABEL_INDEX_CREATE: + case Marker::DELTA_LABEL_INDEX_DROP: + case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: + case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: + case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: + case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: case Marker::VALUE_FALSE: case Marker::VALUE_TRUE: return false; @@ -467,6 +501,51 @@ const uint64_t kVersion{12}; // * number of edges // * number of vertices +// WAL format: +// +// 1) Magic string (non-encoded) +// +// 2) WAL version (non-encoded, little-endian) +// +// 3) Section offsets: +// * offset to the metadata section +// * offset to the first delta in the WAL +// +// 4) Metadata +// * storage UUID +// * sequence number (number indicating the sequence position of this WAL +// file) +// +// 5) Encoded deltas; each delta is written in the following format: +// * commit timestamp +// * action (only one of the actions below are encoded) +// * vertex create, vertex delete +// * gid +// * vertex add label, vertex remove label +// * gid +// * label name +// * vertex set property +// * gid +// * property name +// * property value +// * edge create, edge delete +// * gid +// * edge type name +// * from vertex gid +// * to vertex gid +// * edge set property +// * gid +// * property name +// * property value +// * transaction end (marks that the whole transaction is +// stored in the WAL file) +// * label index create, label index drop +// * label name +// * label property index create, label property index drop, +// existence constraint create, existence constraint drop +// * label name +// * property name + // This is the prefix used for Snapshot and WAL filenames. It is a timestamp // format that equals to: YYYYmmddHHMMSSffffff const std::string kTimestampFormat = @@ -478,6 +557,63 @@ std::string MakeSnapshotName(uint64_t start_timestamp) { std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat); return date_str + "_timestamp_" + std::to_string(start_timestamp); } + +// Generates the name for a WAL file in a well-defined sortable format. +std::string MakeWalName() { + std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat); + return date_str + "_current"; +} + +// Generates the name for a WAL file in a well-defined sortable format with the +// range of timestamps contained [from, to] appended to the name. +std::string RemakeWalName(const std::string ¤t_name, + uint64_t from_timestamp, uint64_t to_timestamp) { + return current_name.substr(0, current_name.size() - 8) + "_from_" + + std::to_string(from_timestamp) + "_to_" + std::to_string(to_timestamp); +} + +Marker OperationToMarker(StorageGlobalOperation operation) { + switch (operation) { + case StorageGlobalOperation::LABEL_INDEX_CREATE: + return Marker::DELTA_LABEL_INDEX_CREATE; + case StorageGlobalOperation::LABEL_INDEX_DROP: + return Marker::DELTA_LABEL_INDEX_DROP; + case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: + return Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE; + case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: + return Marker::DELTA_LABEL_PROPERTY_INDEX_DROP; + case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: + return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE; + case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: + return Marker::DELTA_EXISTENCE_CONSTRAINT_DROP; + } +} + +Marker VertexActionToMarker(Delta::Action action) { + // When converting a Delta to a WAL delta the logic is inverted. That is + // because the Delta's represent undo actions and we want to store redo + // actions. + switch (action) { + case Delta::Action::DELETE_OBJECT: + return Marker::DELTA_VERTEX_CREATE; + case Delta::Action::RECREATE_OBJECT: + return Marker::DELTA_VERTEX_DELETE; + case Delta::Action::SET_PROPERTY: + return Marker::DELTA_VERTEX_SET_PROPERTY; + case Delta::Action::ADD_LABEL: + return Marker::DELTA_VERTEX_REMOVE_LABEL; + case Delta::Action::REMOVE_LABEL: + return Marker::DELTA_VERTEX_ADD_LABEL; + case Delta::Action::ADD_IN_EDGE: + return Marker::DELTA_EDGE_DELETE; + case Delta::Action::ADD_OUT_EDGE: + return Marker::DELTA_EDGE_DELETE; + case Delta::Action::REMOVE_IN_EDGE: + return Marker::DELTA_EDGE_CREATE; + case Delta::Action::REMOVE_OUT_EDGE: + return Marker::DELTA_EDGE_CREATE; + } +} } // namespace // Function used to read information about the snapshot file. @@ -548,6 +684,380 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) { return info; } +// Function used to read information about the WAL file. +WalInfo ReadWalInfo(const std::filesystem::path &path) { + // Check magic and version. + Decoder wal; + auto version = wal.Initialize(path, kWalMagic); + if (!version) + throw RecoveryFailure("Couldn't read WAL magic and/or version!"); + if (*version != kVersion) throw RecoveryFailure("Invalid WAL version!"); + + // Prepare return value. + WalInfo info; + + // Read offsets. + { + auto marker = wal.ReadMarker(); + if (!marker || *marker != Marker::SECTION_OFFSETS) + throw RecoveryFailure("Invalid WAL data!"); + + auto wal_size = wal.GetSize(); + if (!wal_size) throw RecoveryFailure("Invalid WAL data!"); + + auto read_offset = [&wal, wal_size] { + auto maybe_offset = wal.ReadUint(); + if (!maybe_offset) throw RecoveryFailure("Invalid WAL format!"); + auto offset = *maybe_offset; + if (offset > *wal_size) throw RecoveryFailure("Invalid WAL format!"); + return offset; + }; + + info.offset_metadata = read_offset(); + info.offset_deltas = read_offset(); + } + + // Read metadata. + { + wal.SetPosition(info.offset_metadata); + + auto marker = wal.ReadMarker(); + if (!marker || *marker != Marker::SECTION_METADATA) + throw RecoveryFailure("Invalid WAL data!"); + + auto maybe_uuid = wal.ReadString(); + if (!maybe_uuid) throw RecoveryFailure("Invalid WAL data!"); + info.uuid = std::move(*maybe_uuid); + + auto maybe_seq_num = wal.ReadUint(); + if (!maybe_seq_num) throw RecoveryFailure("Invalid WAL data!"); + info.seq_num = *maybe_seq_num; + } + + // Read deltas. + info.num_deltas = 0; + auto validate_delta = [&wal]() -> std::optional> { + auto marker = wal.ReadMarker(); + if (!marker || *marker != Marker::SECTION_DELTA) return std::nullopt; + + auto timestamp = wal.ReadUint(); + if (!timestamp) return std::nullopt; + + auto action = wal.ReadMarker(); + if (!action) return std::nullopt; + + bool is_transaction_end; + switch (*action) { + // These delta actions are all found inside transactions so they don't + // indicate a transaction end. + case Marker::DELTA_VERTEX_CREATE: + case Marker::DELTA_VERTEX_DELETE: + if (!wal.ReadUint()) return std::nullopt; + is_transaction_end = false; + break; + case Marker::DELTA_VERTEX_ADD_LABEL: + case Marker::DELTA_VERTEX_REMOVE_LABEL: + if (!wal.ReadUint() || !wal.SkipString()) return std::nullopt; + is_transaction_end = false; + break; + case Marker::DELTA_EDGE_CREATE: + case Marker::DELTA_EDGE_DELETE: + if (!wal.ReadUint() || !wal.SkipString() || !wal.ReadUint() || + !wal.ReadUint()) + return std::nullopt; + is_transaction_end = false; + break; + case Marker::DELTA_VERTEX_SET_PROPERTY: + case Marker::DELTA_EDGE_SET_PROPERTY: + if (!wal.ReadUint() || !wal.SkipString() || !wal.SkipPropertyValue()) + return std::nullopt; + is_transaction_end = false; + break; + + // This delta explicitly indicates that a transaction is done. + case Marker::DELTA_TRANSACTION_END: + is_transaction_end = true; + break; + + // These operations aren't transactional and they are encoded only using + // a single delta, so they each individually mark the end of their + // 'transaction'. + case Marker::DELTA_LABEL_INDEX_CREATE: + case Marker::DELTA_LABEL_INDEX_DROP: + if (!wal.SkipString()) return std::nullopt; + is_transaction_end = true; + break; + case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: + case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: + case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: + case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: + if (!wal.SkipString() || !wal.SkipString()) return std::nullopt; + is_transaction_end = true; + break; + + // These markers aren't delta actions. + case Marker::TYPE_NULL: + case Marker::TYPE_BOOL: + case Marker::TYPE_INT: + case Marker::TYPE_DOUBLE: + case Marker::TYPE_STRING: + case Marker::TYPE_LIST: + case Marker::TYPE_MAP: + case Marker::TYPE_PROPERTY_VALUE: + case Marker::SECTION_VERTEX: + case Marker::SECTION_EDGE: + case Marker::SECTION_MAPPER: + case Marker::SECTION_METADATA: + case Marker::SECTION_INDICES: + case Marker::SECTION_CONSTRAINTS: + case Marker::SECTION_DELTA: + case Marker::SECTION_OFFSETS: + case Marker::VALUE_FALSE: + case Marker::VALUE_TRUE: + return std::nullopt; + } + + return {{*timestamp, is_transaction_end}}; + }; + auto size = wal.GetSize(); + // Here we read the whole file and determine the number of valid deltas. A + // delta is valid only if all of its data can be successfully read. This + // allows us to recover data from WAL files that are corrupt at the end (eg. + // because of power loss) but are still valid at the beginning. While reading + // the deltas we only count deltas which are a part of a fully valid + // transaction (indicated by a TRANSACTION_END delta or any other + // non-transactional operation). + std::optional current_timestamp; + uint64_t num_deltas = 0; + while (wal.GetPosition() != size) { + auto ret = validate_delta(); + if (!ret) break; + auto [timestamp, is_end_of_transaction] = *ret; + if (!current_timestamp) current_timestamp = timestamp; + if (*current_timestamp != timestamp) break; + ++num_deltas; + if (is_end_of_transaction) { + if (info.num_deltas == 0) { + info.from_timestamp = timestamp; + info.to_timestamp = timestamp; + } + if (timestamp < info.from_timestamp || timestamp < info.to_timestamp) + break; + info.to_timestamp = timestamp; + info.num_deltas += num_deltas; + current_timestamp = std::nullopt; + num_deltas = 0; + } + } + + if (info.num_deltas == 0) throw RecoveryFailure("Invalid WAL data!"); + + return info; +} + +WalFile::WalFile(const std::filesystem::path &wal_directory, + const std::string &uuid, Config::Items items, + NameIdMapper *name_id_mapper, uint64_t seq_num) + : items_(items), + name_id_mapper_(name_id_mapper), + path_(wal_directory / MakeWalName()), + from_timestamp_(0), + to_timestamp_(0), + count_(0) { + // Ensure that the storage directory exists. + utils::EnsureDirOrDie(wal_directory); + + // Initialize the WAL file. + wal_.Initialize(path_, kWalMagic, kVersion); + + // Write placeholder offsets. + uint64_t offset_offsets = 0; + uint64_t offset_metadata = 0; + uint64_t offset_deltas = 0; + wal_.WriteMarker(Marker::SECTION_OFFSETS); + offset_offsets = wal_.GetPosition(); + wal_.WriteUint(offset_metadata); + wal_.WriteUint(offset_deltas); + + // Write metadata. + offset_metadata = wal_.GetPosition(); + wal_.WriteMarker(Marker::SECTION_METADATA); + wal_.WriteString(uuid); + wal_.WriteUint(seq_num); + + // Write final offsets. + offset_deltas = wal_.GetPosition(); + wal_.SetPosition(offset_offsets); + wal_.WriteUint(offset_metadata); + wal_.WriteUint(offset_deltas); + wal_.SetPosition(offset_deltas); + + // Sync the initial data. + wal_.Sync(); +} + +WalFile::~WalFile() { + if (count_ != 0) { + // Finalize file. + wal_.Finalize(); + + // Rename file. + std::filesystem::path new_path(path_); + new_path.replace_filename( + RemakeWalName(path_.filename(), from_timestamp_, to_timestamp_)); + // If the rename fails it isn't a crucial situation. The renaming is done + // only to make the directory structure of the WAL files easier to read + // manually. + utils::RenamePath(path_, new_path); + } else { + // Remove empty WAL file. + utils::DeleteFile(path_); + } +} + +void WalFile::AppendDelta(const Delta &delta, Vertex *vertex, + uint64_t timestamp) { + // When converting a Delta to a WAL delta the logic is inverted. That is + // because the Delta's represent undo actions and we want to store redo + // actions. + wal_.WriteMarker(Marker::SECTION_DELTA); + wal_.WriteUint(timestamp); + std::lock_guard guard(vertex->lock); + switch (delta.action) { + case Delta::Action::DELETE_OBJECT: + case Delta::Action::RECREATE_OBJECT: { + wal_.WriteMarker(VertexActionToMarker(delta.action)); + wal_.WriteUint(vertex->gid.AsUint()); + break; + } + case Delta::Action::SET_PROPERTY: { + wal_.WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY); + wal_.WriteUint(vertex->gid.AsUint()); + wal_.WriteString(name_id_mapper_->IdToName(delta.property.key.AsUint())); + // The property value is the value that is currently stored in the + // vertex. + auto it = vertex->properties.find(delta.property.key); + if (it != vertex->properties.end()) { + wal_.WritePropertyValue(it->second); + } else { + wal_.WritePropertyValue(PropertyValue()); + } + break; + } + case Delta::Action::ADD_LABEL: + case Delta::Action::REMOVE_LABEL: { + wal_.WriteMarker(VertexActionToMarker(delta.action)); + wal_.WriteUint(vertex->gid.AsUint()); + wal_.WriteString(name_id_mapper_->IdToName(delta.label.AsUint())); + break; + } + case Delta::Action::ADD_OUT_EDGE: + case Delta::Action::REMOVE_OUT_EDGE: { + wal_.WriteMarker(VertexActionToMarker(delta.action)); + if (items_.properties_on_edges) { + wal_.WriteUint(delta.vertex_edge.edge.ptr->gid.AsUint()); + } else { + wal_.WriteUint(delta.vertex_edge.edge.gid.AsUint()); + } + wal_.WriteString( + name_id_mapper_->IdToName(delta.vertex_edge.edge_type.AsUint())); + wal_.WriteUint(vertex->gid.AsUint()); + wal_.WriteUint(delta.vertex_edge.vertex->gid.AsUint()); + break; + } + case Delta::Action::ADD_IN_EDGE: + case Delta::Action::REMOVE_IN_EDGE: + // These actions are already encoded in the *_OUT_EDGE actions. This + // function should never be called for this type of deltas. + LOG(FATAL) << "Invalid delta action!"; + } + UpdateStats(timestamp); +} + +void WalFile::AppendDelta(const Delta &delta, Edge *edge, uint64_t timestamp) { + // When converting a Delta to a WAL delta the logic is inverted. That is + // because the Delta's represent undo actions and we want to store redo + // actions. + wal_.WriteMarker(Marker::SECTION_DELTA); + wal_.WriteUint(timestamp); + std::lock_guard guard(edge->lock); + switch (delta.action) { + case Delta::Action::SET_PROPERTY: { + wal_.WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY); + wal_.WriteUint(edge->gid.AsUint()); + wal_.WriteString(name_id_mapper_->IdToName(delta.property.key.AsUint())); + // The property value is the value that is currently stored in the + // edge. + auto it = edge->properties.find(delta.property.key); + if (it != edge->properties.end()) { + wal_.WritePropertyValue(it->second); + } else { + wal_.WritePropertyValue(PropertyValue()); + } + break; + } + case Delta::Action::DELETE_OBJECT: + case Delta::Action::RECREATE_OBJECT: + // These actions are already encoded in vertex *_OUT_EDGE actions. Also, + // these deltas don't contain any information about the from vertex, to + // vertex or edge type so they are useless. This function should never + // be called for this type of deltas. + LOG(FATAL) << "Invalid delta action!"; + case Delta::Action::ADD_LABEL: + case Delta::Action::REMOVE_LABEL: + case Delta::Action::ADD_OUT_EDGE: + case Delta::Action::REMOVE_OUT_EDGE: + case Delta::Action::ADD_IN_EDGE: + case Delta::Action::REMOVE_IN_EDGE: + // These deltas shouldn't appear for edges. + LOG(FATAL) << "Invalid database state!"; + } + UpdateStats(timestamp); +} + +void WalFile::AppendTransactionEnd(uint64_t timestamp) { + wal_.WriteMarker(Marker::SECTION_DELTA); + wal_.WriteUint(timestamp); + wal_.WriteMarker(Marker::DELTA_TRANSACTION_END); + UpdateStats(timestamp); +} + +void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label, + std::optional property, + uint64_t timestamp) { + wal_.WriteMarker(Marker::SECTION_DELTA); + wal_.WriteUint(timestamp); + switch (operation) { + case StorageGlobalOperation::LABEL_INDEX_CREATE: + case StorageGlobalOperation::LABEL_INDEX_DROP: { + wal_.WriteMarker(OperationToMarker(operation)); + wal_.WriteString(name_id_mapper_->IdToName(label.AsUint())); + break; + } + case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: + case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: + case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: + case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: { + CHECK(property) << "Invalid function call!"; + wal_.WriteMarker(OperationToMarker(operation)); + wal_.WriteString(name_id_mapper_->IdToName(label.AsUint())); + wal_.WriteString(name_id_mapper_->IdToName(property->AsUint())); + break; + } + } + UpdateStats(timestamp); +} + +void WalFile::Sync() { wal_.Sync(); } + +uint64_t WalFile::GetSize() { return wal_.GetPosition(); } + +void WalFile::UpdateStats(uint64_t timestamp) { + if (count_ == 0) from_timestamp_ = timestamp; + to_timestamp_ = timestamp; + count_ += 1; +} + Durability::Durability(Config::Durability config, utils::SkipList *vertices, utils::SkipList *edges, diff --git a/src/storage/v2/durability.hpp b/src/storage/v2/durability.hpp index a51ff44f0..8430d0579 100644 --- a/src/storage/v2/durability.hpp +++ b/src/storage/v2/durability.hpp @@ -10,6 +10,7 @@ #include "storage/v2/config.hpp" #include "storage/v2/constraints.hpp" +#include "storage/v2/delta.hpp" #include "storage/v2/edge.hpp" #include "storage/v2/indices.hpp" #include "storage/v2/name_id_mapper.hpp" @@ -47,8 +48,25 @@ enum class Marker : uint8_t { SECTION_METADATA = 0x23, SECTION_INDICES = 0x24, SECTION_CONSTRAINTS = 0x25, + SECTION_DELTA = 0x26, SECTION_OFFSETS = 0x42, + DELTA_VERTEX_CREATE = 0x50, + DELTA_VERTEX_DELETE = 0x51, + DELTA_VERTEX_ADD_LABEL = 0x52, + DELTA_VERTEX_REMOVE_LABEL = 0x53, + DELTA_VERTEX_SET_PROPERTY = 0x54, + DELTA_EDGE_CREATE = 0x55, + DELTA_EDGE_DELETE = 0x56, + DELTA_EDGE_SET_PROPERTY = 0x57, + DELTA_TRANSACTION_END = 0x58, + DELTA_LABEL_INDEX_CREATE = 0x59, + DELTA_LABEL_INDEX_DROP = 0x5a, + DELTA_LABEL_PROPERTY_INDEX_CREATE = 0x5b, + DELTA_LABEL_PROPERTY_INDEX_DROP = 0x5c, + DELTA_EXISTENCE_CONSTRAINT_CREATE = 0x5d, + DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e, + VALUE_FALSE = 0x00, VALUE_TRUE = 0xff, }; @@ -56,14 +74,38 @@ enum class Marker : uint8_t { /// List of all available markers. /// IMPORTANT: Don't forget to update this list when you add a new Marker. static const Marker kMarkersAll[] = { - Marker::TYPE_NULL, Marker::TYPE_BOOL, - Marker::TYPE_INT, Marker::TYPE_DOUBLE, - Marker::TYPE_STRING, Marker::TYPE_LIST, - Marker::TYPE_MAP, Marker::TYPE_PROPERTY_VALUE, - Marker::SECTION_VERTEX, Marker::SECTION_EDGE, - Marker::SECTION_MAPPER, Marker::SECTION_METADATA, - Marker::SECTION_INDICES, Marker::SECTION_CONSTRAINTS, - Marker::SECTION_OFFSETS, Marker::VALUE_FALSE, + Marker::TYPE_NULL, + Marker::TYPE_BOOL, + Marker::TYPE_INT, + Marker::TYPE_DOUBLE, + Marker::TYPE_STRING, + Marker::TYPE_LIST, + Marker::TYPE_MAP, + Marker::TYPE_PROPERTY_VALUE, + Marker::SECTION_VERTEX, + Marker::SECTION_EDGE, + Marker::SECTION_MAPPER, + Marker::SECTION_METADATA, + Marker::SECTION_INDICES, + Marker::SECTION_CONSTRAINTS, + Marker::SECTION_DELTA, + Marker::SECTION_OFFSETS, + Marker::DELTA_VERTEX_CREATE, + Marker::DELTA_VERTEX_DELETE, + Marker::DELTA_VERTEX_ADD_LABEL, + Marker::DELTA_VERTEX_REMOVE_LABEL, + Marker::DELTA_VERTEX_SET_PROPERTY, + Marker::DELTA_EDGE_CREATE, + Marker::DELTA_EDGE_DELETE, + Marker::DELTA_EDGE_SET_PROPERTY, + Marker::DELTA_TRANSACTION_END, + Marker::DELTA_LABEL_INDEX_CREATE, + Marker::DELTA_LABEL_INDEX_DROP, + Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE, + Marker::DELTA_LABEL_PROPERTY_INDEX_DROP, + Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE, + Marker::DELTA_EXISTENCE_CONSTRAINT_DROP, + Marker::VALUE_FALSE, Marker::VALUE_TRUE, }; @@ -87,6 +129,8 @@ class Encoder final { uint64_t GetPosition(); void SetPosition(uint64_t position); + void Sync(); + void Finalize(); private: @@ -148,6 +192,69 @@ struct SnapshotInfo { /// @throw RecoveryFailure SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path); +/// Structure used to hold information about a WAL. +struct WalInfo { + uint64_t offset_metadata; + uint64_t offset_deltas; + + std::string uuid; + uint64_t seq_num; + uint64_t from_timestamp; + uint64_t to_timestamp; + uint64_t num_deltas; +}; + +/// Function used to read information about the WAL file. +/// @throw RecoveryFailure +WalInfo ReadWalInfo(const std::filesystem::path &path); + +/// Enum used to indicate a global database operation that isn't transactional. +enum class StorageGlobalOperation { + LABEL_INDEX_CREATE, + LABEL_INDEX_DROP, + LABEL_PROPERTY_INDEX_CREATE, + LABEL_PROPERTY_INDEX_DROP, + EXISTENCE_CONSTRAINT_CREATE, + EXISTENCE_CONSTRAINT_DROP, +}; + +/// WalFile class used to append deltas and operations to the WAL file. +class WalFile { + public: + WalFile(const std::filesystem::path &wal_directory, const std::string &uuid, + Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num); + + WalFile(const WalFile &) = delete; + WalFile(WalFile &&) = delete; + WalFile &operator=(const WalFile &) = delete; + WalFile &operator=(WalFile &&) = delete; + + ~WalFile(); + + void AppendDelta(const Delta &delta, Vertex *vertex, uint64_t timestamp); + void AppendDelta(const Delta &delta, Edge *edge, uint64_t timestamp); + + void AppendTransactionEnd(uint64_t timestamp); + + void AppendOperation(StorageGlobalOperation operation, LabelId label, + std::optional property, uint64_t timestamp); + + void Sync(); + + uint64_t GetSize(); + + private: + void UpdateStats(uint64_t timestamp); + + Config::Items items_; + NameIdMapper *name_id_mapper_; + Encoder wal_; + std::filesystem::path path_; + uint64_t from_timestamp_; + uint64_t to_timestamp_; + uint64_t count_; +}; + /// Durability class that is used to provide full durability functionality to /// the storage. class Durability final { diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 3ad5904e7..0d59ecf32 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -28,9 +28,6 @@ namespace storage { // The paper implements a fully serializable storage, in our implementation we // only implement snapshot isolation for transactions. -const uint64_t kTimestampInitialId = 0; -const uint64_t kTransactionInitialId = 1ULL << 63U; - /// Iterable for iterating through all vertices of a Storage. /// /// An instance of this will be usually be wrapped inside VerticesIterable for diff --git a/src/storage/v2/transaction.hpp b/src/storage/v2/transaction.hpp index 87610f754..22eb14a9c 100644 --- a/src/storage/v2/transaction.hpp +++ b/src/storage/v2/transaction.hpp @@ -15,6 +15,9 @@ namespace storage { +const uint64_t kTimestampInitialId = 0; +const uint64_t kTransactionInitialId = 1ULL << 63U; + struct Transaction { Transaction(uint64_t transaction_id, uint64_t start_timestamp) : transaction_id(transaction_id), diff --git a/src/utils/file.cpp b/src/utils/file.cpp index 1eb72cf01..280992360 100644 --- a/src/utils/file.cpp +++ b/src/utils/file.cpp @@ -61,6 +61,13 @@ bool CopyFile(const std::filesystem::path &src, return std::filesystem::copy_file(src, dst, error_code); } +bool RenamePath(const std::filesystem::path &src, + const std::filesystem::path &dst) { + std::error_code error_code; // For exception suppression. + std::filesystem::rename(src, dst, error_code); + return !error_code; +} + static_assert(std::is_same_v, "off_t must fit into ssize_t!"); InputFile::~InputFile() { Close(); } diff --git a/src/utils/file.hpp b/src/utils/file.hpp index a6b1b743e..8a1ed519b 100644 --- a/src/utils/file.hpp +++ b/src/utils/file.hpp @@ -40,6 +40,11 @@ bool DeleteFile(const std::filesystem::path &file) noexcept; bool CopyFile(const std::filesystem::path &src, const std::filesystem::path &dst) noexcept; +/// Renames the path from `src` to `dst`. If the `dst` contains directories that +/// don't exist, the renaming fails. Symlinks are not followed. +bool RenamePath(const std::filesystem::path &src, + const std::filesystem::path &dst); + /// This class implements a file handler that is used to read binary files. It /// was developed because the C++ standard library has an awful API and makes /// handling of binary data extremely tedious. diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 5b7e71321..25fd93a65 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -362,6 +362,9 @@ target_link_libraries(${test_prefix}storage_v2_indices mg-storage-v2) add_unit_test(storage_v2_name_id_mapper.cpp) target_link_libraries(${test_prefix}storage_v2_name_id_mapper mg-storage-v2) +add_unit_test(storage_v2_wal_file.cpp) +target_link_libraries(${test_prefix}storage_v2_wal_file mg-storage-v2 fmt) + # Test LCP add_custom_command( diff --git a/tests/unit/storage_v2_decoder_encoder.cpp b/tests/unit/storage_v2_decoder_encoder.cpp index 2643c63f7..63ad13394 100644 --- a/tests/unit/storage_v2_decoder_encoder.cpp +++ b/tests/unit/storage_v2_decoder_encoder.cpp @@ -322,7 +322,23 @@ TEST_F(DecoderEncoderTest, PropertyValueInvalidMarker) { case storage::Marker::SECTION_METADATA: case storage::Marker::SECTION_INDICES: case storage::Marker::SECTION_CONSTRAINTS: + case storage::Marker::SECTION_DELTA: case storage::Marker::SECTION_OFFSETS: + case storage::Marker::DELTA_VERTEX_CREATE: + case storage::Marker::DELTA_VERTEX_DELETE: + case storage::Marker::DELTA_VERTEX_ADD_LABEL: + case storage::Marker::DELTA_VERTEX_REMOVE_LABEL: + case storage::Marker::DELTA_VERTEX_SET_PROPERTY: + case storage::Marker::DELTA_EDGE_CREATE: + case storage::Marker::DELTA_EDGE_DELETE: + case storage::Marker::DELTA_EDGE_SET_PROPERTY: + case storage::Marker::DELTA_TRANSACTION_END: + case storage::Marker::DELTA_LABEL_INDEX_CREATE: + case storage::Marker::DELTA_LABEL_INDEX_DROP: + case storage::Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: + case storage::Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: + case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: + case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: case storage::Marker::VALUE_FALSE: case storage::Marker::VALUE_TRUE: valid_marker = false; diff --git a/tests/unit/storage_v2_wal_file.cpp b/tests/unit/storage_v2_wal_file.cpp new file mode 100644 index 000000000..5755164b2 --- /dev/null +++ b/tests/unit/storage_v2_wal_file.cpp @@ -0,0 +1,506 @@ +#include + +#include + +#include +#include +#include + +#include "storage/v2/durability.hpp" +#include "storage/v2/mvcc.hpp" +#include "storage/v2/name_id_mapper.hpp" +#include "utils/file.hpp" +#include "utils/uuid.hpp" + +// This class mimics the internals of the storage to generate the deltas. +class DeltaGenerator final { + public: + class Transaction final { + private: + friend class DeltaGenerator; + + explicit Transaction(DeltaGenerator *gen) + : gen_(gen), transaction_(gen->transaction_id_++, gen->timestamp_++) {} + + public: + storage::Vertex *CreateVertex() { + auto gid = storage::Gid::FromUint(gen_->vertices_count_++); + auto delta = storage::CreateDeleteObjectDelta(&transaction_); + auto &it = gen_->vertices_.emplace_back(gid, delta); + delta->prev.Set(&it); + return ⁢ + } + + void DeleteVertex(storage::Vertex *vertex) { + storage::CreateAndLinkDelta(&transaction_, &*vertex, + storage::Delta::RecreateObjectTag()); + } + + void AddLabel(storage::Vertex *vertex, const std::string &label) { + auto label_id = storage::LabelId::FromUint(gen_->mapper_.NameToId(label)); + vertex->labels.push_back(label_id); + storage::CreateAndLinkDelta(&transaction_, &*vertex, + storage::Delta::RemoveLabelTag(), label_id); + } + + void RemoveLabel(storage::Vertex *vertex, const std::string &label) { + auto label_id = storage::LabelId::FromUint(gen_->mapper_.NameToId(label)); + vertex->labels.erase( + std::find(vertex->labels.begin(), vertex->labels.end(), label_id)); + storage::CreateAndLinkDelta(&transaction_, &*vertex, + storage::Delta::AddLabelTag(), label_id); + } + + void SetProperty(storage::Vertex *vertex, const std::string &property, + const storage::PropertyValue &value) { + auto property_id = + storage::PropertyId::FromUint(gen_->mapper_.NameToId(property)); + auto &props = vertex->properties; + auto it = props.find(property_id); + if (it == props.end()) { + storage::CreateAndLinkDelta(&transaction_, &*vertex, + storage::Delta::SetPropertyTag(), + property_id, storage::PropertyValue()); + if (!value.IsNull()) { + props.emplace(property_id, value); + } + } else { + storage::CreateAndLinkDelta(&transaction_, &*vertex, + storage::Delta::SetPropertyTag(), + property_id, it->second); + if (!value.IsNull()) { + it->second = value; + } else { + props.erase(it); + } + } + } + + void Finalize(bool append_transaction_end = true) { + auto commit_timestamp = gen_->timestamp_++; + for (const auto &delta : transaction_.deltas) { + auto owner = delta.prev.Get(); + while (owner.type == storage::PreviousPtr::Type::DELTA) { + owner = owner.delta->prev.Get(); + } + if (owner.type == storage::PreviousPtr::Type::VERTEX) { + gen_->wal_file_.AppendDelta(delta, owner.vertex, commit_timestamp); + } else if (owner.type == storage::PreviousPtr::Type::EDGE) { + gen_->wal_file_.AppendDelta(delta, owner.edge, commit_timestamp); + } else { + LOG(FATAL) << "Invalid delta owner!"; + } + } + if (append_transaction_end) { + gen_->wal_file_.AppendTransactionEnd(commit_timestamp); + gen_->UpdateStats(commit_timestamp, transaction_.deltas.size() + 1); + } else { + gen_->valid_ = false; + } + } + + private: + DeltaGenerator *gen_; + storage::Transaction transaction_; + }; + + DeltaGenerator(const std::filesystem::path &data_directory, + bool properties_on_edges, uint64_t seq_num) + : uuid_(utils::GenerateUUID()), + seq_num_(seq_num), + wal_file_(data_directory, uuid_, + {.properties_on_edges = properties_on_edges}, &mapper_, + seq_num) {} + + Transaction CreateTransaction() { return Transaction(this); } + + void ResetTransactionIds() { + transaction_id_ = storage::kTransactionInitialId; + timestamp_ = storage::kTimestampInitialId; + valid_ = false; + } + + void AppendOperation(storage::StorageGlobalOperation operation, + const std::string &label, + std::optional property = std::nullopt) { + auto label_id = storage::LabelId::FromUint(mapper_.NameToId(label)); + std::optional property_id; + if (property) { + property_id = storage::PropertyId::FromUint(mapper_.NameToId(*property)); + } + wal_file_.AppendOperation(operation, label_id, property_id, timestamp_); + UpdateStats(timestamp_, 1); + } + + uint64_t GetPosition() { return wal_file_.GetSize(); } + + storage::WalInfo GetInfo() { + return {.offset_metadata = 0, + .offset_deltas = 0, + .uuid = uuid_, + .seq_num = seq_num_, + .from_timestamp = tx_from_, + .to_timestamp = tx_to_, + .num_deltas = deltas_count_}; + } + + private: + void UpdateStats(uint64_t timestamp, uint64_t count) { + if (!valid_) return; + if (deltas_count_ == 0) { + tx_from_ = timestamp; + } + tx_to_ = timestamp; + deltas_count_ += count; + } + + std::string uuid_; + uint64_t seq_num_; + + uint64_t transaction_id_{storage::kTransactionInitialId}; + uint64_t timestamp_{storage::kTimestampInitialId}; + uint64_t vertices_count_{0}; + std::list vertices_; + storage::NameIdMapper mapper_; + + storage::WalFile wal_file_; + + uint64_t deltas_count_{0}; + uint64_t tx_from_{0}; + uint64_t tx_to_{0}; + uint64_t valid_{true}; +}; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define TRANSACTION(append_transaction_end, ops) \ + { \ + auto tx = gen.CreateTransaction(); \ + ops; \ + tx.Finalize(append_transaction_end); \ + } + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define OPERATION(op, ...) \ + gen.AppendOperation(storage::StorageGlobalOperation::op, __VA_ARGS__) + +void AssertWalInfoEqual(const storage::WalInfo &a, const storage::WalInfo &b) { + ASSERT_EQ(a.uuid, b.uuid); + ASSERT_EQ(a.seq_num, b.seq_num); + ASSERT_EQ(a.from_timestamp, b.from_timestamp); + ASSERT_EQ(a.to_timestamp, b.to_timestamp); + ASSERT_EQ(a.num_deltas, b.num_deltas); +} + +class WalFileTest : public ::testing::TestWithParam { + public: + WalFileTest() {} + + void SetUp() override { Clear(); } + + void TearDown() override { Clear(); } + + std::vector GetFilesList() { + std::vector ret; + for (auto &item : std::filesystem::directory_iterator(storage_directory)) { + ret.push_back(item.path()); + } + std::sort(ret.begin(), ret.end()); + std::reverse(ret.begin(), ret.end()); + return ret; + } + + std::filesystem::path storage_directory{ + std::filesystem::temp_directory_path() / + "MG_test_unit_storage_v2_wal_file"}; + + private: + void Clear() { + if (!std::filesystem::exists(storage_directory)) return; + std::filesystem::remove_all(storage_directory); + } +}; + +INSTANTIATE_TEST_CASE_P(EdgesWithProperties, WalFileTest, + ::testing::Values(true)); +INSTANTIATE_TEST_CASE_P(EdgesWithoutProperties, WalFileTest, + ::testing::Values(false)); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(WalFileTest, EmptyFile) { + { DeltaGenerator gen(storage_directory, GetParam(), 5); } + auto wal_files = GetFilesList(); + ASSERT_EQ(wal_files.size(), 0); +} + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define GENERATE_SIMPLE_TEST(name, ops) \ + TEST_P(WalFileTest, name) { \ + storage::WalInfo info; \ + \ + { \ + DeltaGenerator gen(storage_directory, GetParam(), 5); \ + ops; \ + info = gen.GetInfo(); \ + } \ + \ + auto wal_files = GetFilesList(); \ + ASSERT_EQ(wal_files.size(), 1); \ + \ + if (info.num_deltas == 0) { \ + ASSERT_THROW(storage::ReadWalInfo(wal_files.front()), \ + storage::RecoveryFailure); \ + } else { \ + AssertWalInfoEqual(info, storage::ReadWalInfo(wal_files.front())); \ + } \ + } + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionWithEnd, + { TRANSACTION(true, { tx.CreateVertex(); }); }); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionWithoutEnd, + { TRANSACTION(false, { tx.CreateVertex(); }); }); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(OperationSingle, + { OPERATION(LABEL_INDEX_CREATE, "hello"); }); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsEnd00, { + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsEnd01, { + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsEnd10, { + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsEnd11, { + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation_00, { + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation_01, { + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation_10, { + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation_11, { + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation0_0, { + TRANSACTION(false, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation0_1, { + TRANSACTION(false, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(true, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation1_0, { + TRANSACTION(true, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(false, { tx.CreateVertex(); }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation1_1, { + TRANSACTION(true, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); + TRANSACTION(true, { tx.CreateVertex(); }); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation00_, { + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation01_, { + TRANSACTION(false, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation10_, { + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(false, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(TransactionsWithOperation11_, { + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); + OPERATION(LABEL_INDEX_CREATE, "hello"); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(AllTransactionOperationsWithEnd, { + TRANSACTION(true, { + auto vertex1 = tx.CreateVertex(); + auto vertex2 = tx.CreateVertex(); + tx.AddLabel(vertex1, "test"); + tx.AddLabel(vertex2, "hello"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare")); + tx.RemoveLabel(vertex1, "test"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue(123)); + tx.SetProperty(vertex2, "hello", storage::PropertyValue()); + tx.DeleteVertex(vertex1); + }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(AllTransactionOperationsWithoutEnd, { + TRANSACTION(false, { + auto vertex1 = tx.CreateVertex(); + auto vertex2 = tx.CreateVertex(); + tx.AddLabel(vertex1, "test"); + tx.AddLabel(vertex2, "hello"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare")); + tx.RemoveLabel(vertex1, "test"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue(123)); + tx.SetProperty(vertex2, "hello", storage::PropertyValue()); + tx.DeleteVertex(vertex1); + }); +}); +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(AllGlobalOperations, { + OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION(LABEL_INDEX_DROP, "hello"); + OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world"); + OPERATION(LABEL_PROPERTY_INDEX_DROP, "hello", "world"); + OPERATION(EXISTENCE_CONSTRAINT_CREATE, "hello", "world"); + OPERATION(EXISTENCE_CONSTRAINT_DROP, "hello", "world"); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(InvalidTransactionOrdering, { + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); + TRANSACTION(true, { tx.CreateVertex(); }); + gen.ResetTransactionIds(); + TRANSACTION(true, { tx.CreateVertex(); }); +}); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(WalFileTest, InvalidMarker) { + storage::WalInfo info; + + { + DeltaGenerator gen(storage_directory, GetParam(), 5); + TRANSACTION(true, { tx.CreateVertex(); }); + info = gen.GetInfo(); + } + + auto wal_files = GetFilesList(); + ASSERT_EQ(wal_files.size(), 1); + const auto &wal_file = wal_files.front(); + + auto final_info = storage::ReadWalInfo(wal_file); + AssertWalInfoEqual(info, final_info); + + size_t i = 0; + for (auto marker : storage::kMarkersAll) { + if (marker == storage::Marker::SECTION_DELTA) continue; + auto current_file = storage_directory / fmt::format("temporary_{}", i); + ASSERT_TRUE(std::filesystem::copy_file(wal_file, current_file)); + utils::OutputFile file; + file.Open(current_file, utils::OutputFile::Mode::OVERWRITE_EXISTING); + file.SetPosition(utils::OutputFile::Position::SET, + final_info.offset_deltas); + auto value = static_cast(marker); + file.Write(&value, sizeof(value)); + file.Sync(); + file.Close(); + ASSERT_THROW(storage::ReadWalInfo(current_file), storage::RecoveryFailure); + ++i; + } +} + +// NOLINTNEXTLINE(hicpp-special-member-functions) +TEST_P(WalFileTest, PartialData) { + std::vector> infos; + + { + DeltaGenerator gen(storage_directory, GetParam(), 5); + TRANSACTION(true, { tx.CreateVertex(); }); + infos.emplace_back(gen.GetPosition(), gen.GetInfo()); + TRANSACTION(true, { + auto vertex = tx.CreateVertex(); + tx.AddLabel(vertex, "hello"); + }); + infos.emplace_back(gen.GetPosition(), gen.GetInfo()); + OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world"); + infos.emplace_back(gen.GetPosition(), gen.GetInfo()); + TRANSACTION(true, { + auto vertex1 = tx.CreateVertex(); + auto vertex2 = tx.CreateVertex(); + tx.AddLabel(vertex1, "test"); + tx.AddLabel(vertex2, "hello"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare")); + tx.RemoveLabel(vertex1, "test"); + tx.SetProperty(vertex2, "hello", storage::PropertyValue(123)); + tx.SetProperty(vertex2, "hello", storage::PropertyValue()); + tx.DeleteVertex(vertex1); + }); + infos.emplace_back(gen.GetPosition(), gen.GetInfo()); + } + + auto wal_files = GetFilesList(); + ASSERT_EQ(wal_files.size(), 1); + const auto &wal_file = wal_files.front(); + + AssertWalInfoEqual(infos.back().second, storage::ReadWalInfo(wal_file)); + + auto current_file = storage_directory / "temporary"; + utils::InputFile infile; + infile.Open(wal_file); + + uint64_t pos = 0; + for (size_t i = 0; i < infile.GetSize(); ++i) { + if (i < infos.front().first) { + ASSERT_THROW(storage::ReadWalInfo(current_file), + storage::RecoveryFailure); + } else { + if (i >= infos[pos + 1].first) ++pos; + AssertWalInfoEqual(infos[pos].second, storage::ReadWalInfo(current_file)); + } + { + utils::OutputFile outfile; + outfile.Open(current_file, utils::OutputFile::Mode::APPEND_TO_EXISTING); + uint8_t value; + ASSERT_TRUE(infile.Read(&value, sizeof(value))); + outfile.Write(&value, sizeof(value)); + outfile.Sync(); + outfile.Close(); + } + } + ASSERT_EQ(pos, infos.size() - 2); + AssertWalInfoEqual(infos[infos.size() - 1].second, + storage::ReadWalInfo(current_file)); +}