Implement WalFile for storage v2
Reviewers: teon.banek Reviewed By: teon.banek Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D2510
This commit is contained in:
Matej Ferencevic
parent
d2fac02b74
commit
98855889d5
@ -123,6 +123,8 @@ void Encoder::SetPosition(uint64_t position) {
|
||||
file_.SetPosition(utils::OutputFile::Position::SET, position);
|
||||
}
|
||||
|
||||
void Encoder::Sync() { file_.Sync(); }
|
||||
|
||||
void Encoder::Finalize() {
|
||||
file_.Sync();
|
||||
file_.Close();
|
||||
@ -303,7 +305,23 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
|
||||
case Marker::SECTION_METADATA:
|
||||
case Marker::SECTION_INDICES:
|
||||
case Marker::SECTION_CONSTRAINTS:
|
||||
case Marker::SECTION_DELTA:
|
||||
case Marker::SECTION_OFFSETS:
|
||||
case Marker::DELTA_VERTEX_CREATE:
|
||||
case Marker::DELTA_VERTEX_DELETE:
|
||||
case Marker::DELTA_VERTEX_ADD_LABEL:
|
||||
case Marker::DELTA_VERTEX_REMOVE_LABEL:
|
||||
case Marker::DELTA_VERTEX_SET_PROPERTY:
|
||||
case Marker::DELTA_EDGE_CREATE:
|
||||
case Marker::DELTA_EDGE_DELETE:
|
||||
case Marker::DELTA_EDGE_SET_PROPERTY:
|
||||
case Marker::DELTA_TRANSACTION_END:
|
||||
case Marker::DELTA_LABEL_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_INDEX_DROP:
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
|
||||
case Marker::VALUE_FALSE:
|
||||
case Marker::VALUE_TRUE:
|
||||
return std::nullopt;
|
||||
@ -380,7 +398,23 @@ bool Decoder::SkipPropertyValue() {
|
||||
case Marker::SECTION_METADATA:
|
||||
case Marker::SECTION_INDICES:
|
||||
case Marker::SECTION_CONSTRAINTS:
|
||||
case Marker::SECTION_DELTA:
|
||||
case Marker::SECTION_OFFSETS:
|
||||
case Marker::DELTA_VERTEX_CREATE:
|
||||
case Marker::DELTA_VERTEX_DELETE:
|
||||
case Marker::DELTA_VERTEX_ADD_LABEL:
|
||||
case Marker::DELTA_VERTEX_REMOVE_LABEL:
|
||||
case Marker::DELTA_VERTEX_SET_PROPERTY:
|
||||
case Marker::DELTA_EDGE_CREATE:
|
||||
case Marker::DELTA_EDGE_DELETE:
|
||||
case Marker::DELTA_EDGE_SET_PROPERTY:
|
||||
case Marker::DELTA_TRANSACTION_END:
|
||||
case Marker::DELTA_LABEL_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_INDEX_DROP:
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
|
||||
case Marker::VALUE_FALSE:
|
||||
case Marker::VALUE_TRUE:
|
||||
return false;
|
||||
@ -467,6 +501,51 @@ const uint64_t kVersion{12};
|
||||
// * number of edges
|
||||
// * number of vertices
|
||||
|
||||
// WAL format:
|
||||
//
|
||||
// 1) Magic string (non-encoded)
|
||||
//
|
||||
// 2) WAL version (non-encoded, little-endian)
|
||||
//
|
||||
// 3) Section offsets:
|
||||
// * offset to the metadata section
|
||||
// * offset to the first delta in the WAL
|
||||
//
|
||||
// 4) Metadata
|
||||
// * storage UUID
|
||||
// * sequence number (number indicating the sequence position of this WAL
|
||||
// file)
|
||||
//
|
||||
// 5) Encoded deltas; each delta is written in the following format:
|
||||
// * commit timestamp
|
||||
// * action (only one of the actions below are encoded)
|
||||
// * vertex create, vertex delete
|
||||
// * gid
|
||||
// * vertex add label, vertex remove label
|
||||
// * gid
|
||||
// * label name
|
||||
// * vertex set property
|
||||
// * gid
|
||||
// * property name
|
||||
// * property value
|
||||
// * edge create, edge delete
|
||||
// * gid
|
||||
// * edge type name
|
||||
// * from vertex gid
|
||||
// * to vertex gid
|
||||
// * edge set property
|
||||
// * gid
|
||||
// * property name
|
||||
// * property value
|
||||
// * transaction end (marks that the whole transaction is
|
||||
// stored in the WAL file)
|
||||
// * label index create, label index drop
|
||||
// * label name
|
||||
// * label property index create, label property index drop,
|
||||
// existence constraint create, existence constraint drop
|
||||
// * label name
|
||||
// * property name
|
||||
|
||||
// This is the prefix used for Snapshot and WAL filenames. It is a timestamp
|
||||
// format that equals to: YYYYmmddHHMMSSffffff
|
||||
const std::string kTimestampFormat =
|
||||
@ -478,6 +557,63 @@ std::string MakeSnapshotName(uint64_t start_timestamp) {
|
||||
std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat);
|
||||
return date_str + "_timestamp_" + std::to_string(start_timestamp);
|
||||
}
|
||||
|
||||
// Generates the name for a WAL file in a well-defined sortable format.
|
||||
std::string MakeWalName() {
|
||||
std::string date_str = utils::Timestamp::Now().ToString(kTimestampFormat);
|
||||
return date_str + "_current";
|
||||
}
|
||||
|
||||
// Generates the name for a WAL file in a well-defined sortable format with the
|
||||
// range of timestamps contained [from, to] appended to the name.
|
||||
std::string RemakeWalName(const std::string ¤t_name,
|
||||
uint64_t from_timestamp, uint64_t to_timestamp) {
|
||||
return current_name.substr(0, current_name.size() - 8) + "_from_" +
|
||||
std::to_string(from_timestamp) + "_to_" + std::to_string(to_timestamp);
|
||||
}
|
||||
|
||||
Marker OperationToMarker(StorageGlobalOperation operation) {
|
||||
switch (operation) {
|
||||
case StorageGlobalOperation::LABEL_INDEX_CREATE:
|
||||
return Marker::DELTA_LABEL_INDEX_CREATE;
|
||||
case StorageGlobalOperation::LABEL_INDEX_DROP:
|
||||
return Marker::DELTA_LABEL_INDEX_DROP;
|
||||
case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE:
|
||||
return Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE;
|
||||
case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP:
|
||||
return Marker::DELTA_LABEL_PROPERTY_INDEX_DROP;
|
||||
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE:
|
||||
return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE;
|
||||
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP:
|
||||
return Marker::DELTA_EXISTENCE_CONSTRAINT_DROP;
|
||||
}
|
||||
}
|
||||
|
||||
Marker VertexActionToMarker(Delta::Action action) {
|
||||
// When converting a Delta to a WAL delta the logic is inverted. That is
|
||||
// because the Delta's represent undo actions and we want to store redo
|
||||
// actions.
|
||||
switch (action) {
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
return Marker::DELTA_VERTEX_CREATE;
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
return Marker::DELTA_VERTEX_DELETE;
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
return Marker::DELTA_VERTEX_SET_PROPERTY;
|
||||
case Delta::Action::ADD_LABEL:
|
||||
return Marker::DELTA_VERTEX_REMOVE_LABEL;
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
return Marker::DELTA_VERTEX_ADD_LABEL;
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
return Marker::DELTA_EDGE_DELETE;
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
return Marker::DELTA_EDGE_DELETE;
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
return Marker::DELTA_EDGE_CREATE;
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
return Marker::DELTA_EDGE_CREATE;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Function used to read information about the snapshot file.
|
||||
@ -548,6 +684,380 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
|
||||
return info;
|
||||
}
|
||||
|
||||
// Function used to read information about the WAL file.
|
||||
WalInfo ReadWalInfo(const std::filesystem::path &path) {
|
||||
// Check magic and version.
|
||||
Decoder wal;
|
||||
auto version = wal.Initialize(path, kWalMagic);
|
||||
if (!version)
|
||||
throw RecoveryFailure("Couldn't read WAL magic and/or version!");
|
||||
if (*version != kVersion) throw RecoveryFailure("Invalid WAL version!");
|
||||
|
||||
// Prepare return value.
|
||||
WalInfo info;
|
||||
|
||||
// Read offsets.
|
||||
{
|
||||
auto marker = wal.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_OFFSETS)
|
||||
throw RecoveryFailure("Invalid WAL data!");
|
||||
|
||||
auto wal_size = wal.GetSize();
|
||||
if (!wal_size) throw RecoveryFailure("Invalid WAL data!");
|
||||
|
||||
auto read_offset = [&wal, wal_size] {
|
||||
auto maybe_offset = wal.ReadUint();
|
||||
if (!maybe_offset) throw RecoveryFailure("Invalid WAL format!");
|
||||
auto offset = *maybe_offset;
|
||||
if (offset > *wal_size) throw RecoveryFailure("Invalid WAL format!");
|
||||
return offset;
|
||||
};
|
||||
|
||||
info.offset_metadata = read_offset();
|
||||
info.offset_deltas = read_offset();
|
||||
}
|
||||
|
||||
// Read metadata.
|
||||
{
|
||||
wal.SetPosition(info.offset_metadata);
|
||||
|
||||
auto marker = wal.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_METADATA)
|
||||
throw RecoveryFailure("Invalid WAL data!");
|
||||
|
||||
auto maybe_uuid = wal.ReadString();
|
||||
if (!maybe_uuid) throw RecoveryFailure("Invalid WAL data!");
|
||||
info.uuid = std::move(*maybe_uuid);
|
||||
|
||||
auto maybe_seq_num = wal.ReadUint();
|
||||
if (!maybe_seq_num) throw RecoveryFailure("Invalid WAL data!");
|
||||
info.seq_num = *maybe_seq_num;
|
||||
}
|
||||
|
||||
// Read deltas.
|
||||
info.num_deltas = 0;
|
||||
auto validate_delta = [&wal]() -> std::optional<std::pair<uint64_t, bool>> {
|
||||
auto marker = wal.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_DELTA) return std::nullopt;
|
||||
|
||||
auto timestamp = wal.ReadUint();
|
||||
if (!timestamp) return std::nullopt;
|
||||
|
||||
auto action = wal.ReadMarker();
|
||||
if (!action) return std::nullopt;
|
||||
|
||||
bool is_transaction_end;
|
||||
switch (*action) {
|
||||
// These delta actions are all found inside transactions so they don't
|
||||
// indicate a transaction end.
|
||||
case Marker::DELTA_VERTEX_CREATE:
|
||||
case Marker::DELTA_VERTEX_DELETE:
|
||||
if (!wal.ReadUint()) return std::nullopt;
|
||||
is_transaction_end = false;
|
||||
break;
|
||||
case Marker::DELTA_VERTEX_ADD_LABEL:
|
||||
case Marker::DELTA_VERTEX_REMOVE_LABEL:
|
||||
if (!wal.ReadUint() || !wal.SkipString()) return std::nullopt;
|
||||
is_transaction_end = false;
|
||||
break;
|
||||
case Marker::DELTA_EDGE_CREATE:
|
||||
case Marker::DELTA_EDGE_DELETE:
|
||||
if (!wal.ReadUint() || !wal.SkipString() || !wal.ReadUint() ||
|
||||
!wal.ReadUint())
|
||||
return std::nullopt;
|
||||
is_transaction_end = false;
|
||||
break;
|
||||
case Marker::DELTA_VERTEX_SET_PROPERTY:
|
||||
case Marker::DELTA_EDGE_SET_PROPERTY:
|
||||
if (!wal.ReadUint() || !wal.SkipString() || !wal.SkipPropertyValue())
|
||||
return std::nullopt;
|
||||
is_transaction_end = false;
|
||||
break;
|
||||
|
||||
// This delta explicitly indicates that a transaction is done.
|
||||
case Marker::DELTA_TRANSACTION_END:
|
||||
is_transaction_end = true;
|
||||
break;
|
||||
|
||||
// These operations aren't transactional and they are encoded only using
|
||||
// a single delta, so they each individually mark the end of their
|
||||
// 'transaction'.
|
||||
case Marker::DELTA_LABEL_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_INDEX_DROP:
|
||||
if (!wal.SkipString()) return std::nullopt;
|
||||
is_transaction_end = true;
|
||||
break;
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
|
||||
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
|
||||
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
|
||||
if (!wal.SkipString() || !wal.SkipString()) return std::nullopt;
|
||||
is_transaction_end = true;
|
||||
break;
|
||||
|
||||
// These markers aren't delta actions.
|
||||
case Marker::TYPE_NULL:
|
||||
case Marker::TYPE_BOOL:
|
||||
case Marker::TYPE_INT:
|
||||
case Marker::TYPE_DOUBLE:
|
||||
case Marker::TYPE_STRING:
|
||||
case Marker::TYPE_LIST:
|
||||
case Marker::TYPE_MAP:
|
||||
case Marker::TYPE_PROPERTY_VALUE:
|
||||
case Marker::SECTION_VERTEX:
|
||||
case Marker::SECTION_EDGE:
|
||||
case Marker::SECTION_MAPPER:
|
||||
case Marker::SECTION_METADATA:
|
||||
case Marker::SECTION_INDICES:
|
||||
case Marker::SECTION_CONSTRAINTS:
|
||||
case Marker::SECTION_DELTA:
|
||||
case Marker::SECTION_OFFSETS:
|
||||
case Marker::VALUE_FALSE:
|
||||
case Marker::VALUE_TRUE:
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return {{*timestamp, is_transaction_end}};
|
||||
};
|
||||
auto size = wal.GetSize();
|
||||
// Here we read the whole file and determine the number of valid deltas. A
|
||||
// delta is valid only if all of its data can be successfully read. This
|
||||
// allows us to recover data from WAL files that are corrupt at the end (eg.
|
||||
// because of power loss) but are still valid at the beginning. While reading
|
||||
// the deltas we only count deltas which are a part of a fully valid
|
||||
// transaction (indicated by a TRANSACTION_END delta or any other
|
||||
// non-transactional operation).
|
||||
std::optional<uint64_t> current_timestamp;
|
||||
uint64_t num_deltas = 0;
|
||||
while (wal.GetPosition() != size) {
|
||||
auto ret = validate_delta();
|
||||
if (!ret) break;
|
||||
auto [timestamp, is_end_of_transaction] = *ret;
|
||||
if (!current_timestamp) current_timestamp = timestamp;
|
||||
if (*current_timestamp != timestamp) break;
|
||||
++num_deltas;
|
||||
if (is_end_of_transaction) {
|
||||
if (info.num_deltas == 0) {
|
||||
info.from_timestamp = timestamp;
|
||||
info.to_timestamp = timestamp;
|
||||
}
|
||||
if (timestamp < info.from_timestamp || timestamp < info.to_timestamp)
|
||||
break;
|
||||
info.to_timestamp = timestamp;
|
||||
info.num_deltas += num_deltas;
|
||||
current_timestamp = std::nullopt;
|
||||
num_deltas = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (info.num_deltas == 0) throw RecoveryFailure("Invalid WAL data!");
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
WalFile::WalFile(const std::filesystem::path &wal_directory,
|
||||
const std::string &uuid, Config::Items items,
|
||||
NameIdMapper *name_id_mapper, uint64_t seq_num)
|
||||
: items_(items),
|
||||
name_id_mapper_(name_id_mapper),
|
||||
path_(wal_directory / MakeWalName()),
|
||||
from_timestamp_(0),
|
||||
to_timestamp_(0),
|
||||
count_(0) {
|
||||
// Ensure that the storage directory exists.
|
||||
utils::EnsureDirOrDie(wal_directory);
|
||||
|
||||
// Initialize the WAL file.
|
||||
wal_.Initialize(path_, kWalMagic, kVersion);
|
||||
|
||||
// Write placeholder offsets.
|
||||
uint64_t offset_offsets = 0;
|
||||
uint64_t offset_metadata = 0;
|
||||
uint64_t offset_deltas = 0;
|
||||
wal_.WriteMarker(Marker::SECTION_OFFSETS);
|
||||
offset_offsets = wal_.GetPosition();
|
||||
wal_.WriteUint(offset_metadata);
|
||||
wal_.WriteUint(offset_deltas);
|
||||
|
||||
// Write metadata.
|
||||
offset_metadata = wal_.GetPosition();
|
||||
wal_.WriteMarker(Marker::SECTION_METADATA);
|
||||
wal_.WriteString(uuid);
|
||||
wal_.WriteUint(seq_num);
|
||||
|
||||
// Write final offsets.
|
||||
offset_deltas = wal_.GetPosition();
|
||||
wal_.SetPosition(offset_offsets);
|
||||
wal_.WriteUint(offset_metadata);
|
||||
wal_.WriteUint(offset_deltas);
|
||||
wal_.SetPosition(offset_deltas);
|
||||
|
||||
// Sync the initial data.
|
||||
wal_.Sync();
|
||||
}
|
||||
|
||||
WalFile::~WalFile() {
|
||||
if (count_ != 0) {
|
||||
// Finalize file.
|
||||
wal_.Finalize();
|
||||
|
||||
// Rename file.
|
||||
std::filesystem::path new_path(path_);
|
||||
new_path.replace_filename(
|
||||
RemakeWalName(path_.filename(), from_timestamp_, to_timestamp_));
|
||||
// If the rename fails it isn't a crucial situation. The renaming is done
|
||||
// only to make the directory structure of the WAL files easier to read
|
||||
// manually.
|
||||
utils::RenamePath(path_, new_path);
|
||||
} else {
|
||||
// Remove empty WAL file.
|
||||
utils::DeleteFile(path_);
|
||||
}
|
||||
}
|
||||
|
||||
void WalFile::AppendDelta(const Delta &delta, Vertex *vertex,
|
||||
uint64_t timestamp) {
|
||||
// When converting a Delta to a WAL delta the logic is inverted. That is
|
||||
// because the Delta's represent undo actions and we want to store redo
|
||||
// actions.
|
||||
wal_.WriteMarker(Marker::SECTION_DELTA);
|
||||
wal_.WriteUint(timestamp);
|
||||
std::lock_guard<utils::SpinLock> guard(vertex->lock);
|
||||
switch (delta.action) {
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
wal_.WriteMarker(VertexActionToMarker(delta.action));
|
||||
wal_.WriteUint(vertex->gid.AsUint());
|
||||
break;
|
||||
}
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
wal_.WriteMarker(Marker::DELTA_VERTEX_SET_PROPERTY);
|
||||
wal_.WriteUint(vertex->gid.AsUint());
|
||||
wal_.WriteString(name_id_mapper_->IdToName(delta.property.key.AsUint()));
|
||||
// The property value is the value that is currently stored in the
|
||||
// vertex.
|
||||
auto it = vertex->properties.find(delta.property.key);
|
||||
if (it != vertex->properties.end()) {
|
||||
wal_.WritePropertyValue(it->second);
|
||||
} else {
|
||||
wal_.WritePropertyValue(PropertyValue());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
wal_.WriteMarker(VertexActionToMarker(delta.action));
|
||||
wal_.WriteUint(vertex->gid.AsUint());
|
||||
wal_.WriteString(name_id_mapper_->IdToName(delta.label.AsUint()));
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
wal_.WriteMarker(VertexActionToMarker(delta.action));
|
||||
if (items_.properties_on_edges) {
|
||||
wal_.WriteUint(delta.vertex_edge.edge.ptr->gid.AsUint());
|
||||
} else {
|
||||
wal_.WriteUint(delta.vertex_edge.edge.gid.AsUint());
|
||||
}
|
||||
wal_.WriteString(
|
||||
name_id_mapper_->IdToName(delta.vertex_edge.edge_type.AsUint()));
|
||||
wal_.WriteUint(vertex->gid.AsUint());
|
||||
wal_.WriteUint(delta.vertex_edge.vertex->gid.AsUint());
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
// These actions are already encoded in the *_OUT_EDGE actions. This
|
||||
// function should never be called for this type of deltas.
|
||||
LOG(FATAL) << "Invalid delta action!";
|
||||
}
|
||||
UpdateStats(timestamp);
|
||||
}
|
||||
|
||||
void WalFile::AppendDelta(const Delta &delta, Edge *edge, uint64_t timestamp) {
|
||||
// When converting a Delta to a WAL delta the logic is inverted. That is
|
||||
// because the Delta's represent undo actions and we want to store redo
|
||||
// actions.
|
||||
wal_.WriteMarker(Marker::SECTION_DELTA);
|
||||
wal_.WriteUint(timestamp);
|
||||
std::lock_guard<utils::SpinLock> guard(edge->lock);
|
||||
switch (delta.action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
wal_.WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY);
|
||||
wal_.WriteUint(edge->gid.AsUint());
|
||||
wal_.WriteString(name_id_mapper_->IdToName(delta.property.key.AsUint()));
|
||||
// The property value is the value that is currently stored in the
|
||||
// edge.
|
||||
auto it = edge->properties.find(delta.property.key);
|
||||
if (it != edge->properties.end()) {
|
||||
wal_.WritePropertyValue(it->second);
|
||||
} else {
|
||||
wal_.WritePropertyValue(PropertyValue());
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
// These actions are already encoded in vertex *_OUT_EDGE actions. Also,
|
||||
// these deltas don't contain any information about the from vertex, to
|
||||
// vertex or edge type so they are useless. This function should never
|
||||
// be called for this type of deltas.
|
||||
LOG(FATAL) << "Invalid delta action!";
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
// These deltas shouldn't appear for edges.
|
||||
LOG(FATAL) << "Invalid database state!";
|
||||
}
|
||||
UpdateStats(timestamp);
|
||||
}
|
||||
|
||||
void WalFile::AppendTransactionEnd(uint64_t timestamp) {
|
||||
wal_.WriteMarker(Marker::SECTION_DELTA);
|
||||
wal_.WriteUint(timestamp);
|
||||
wal_.WriteMarker(Marker::DELTA_TRANSACTION_END);
|
||||
UpdateStats(timestamp);
|
||||
}
|
||||
|
||||
void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label,
|
||||
std::optional<PropertyId> property,
|
||||
uint64_t timestamp) {
|
||||
wal_.WriteMarker(Marker::SECTION_DELTA);
|
||||
wal_.WriteUint(timestamp);
|
||||
switch (operation) {
|
||||
case StorageGlobalOperation::LABEL_INDEX_CREATE:
|
||||
case StorageGlobalOperation::LABEL_INDEX_DROP: {
|
||||
wal_.WriteMarker(OperationToMarker(operation));
|
||||
wal_.WriteString(name_id_mapper_->IdToName(label.AsUint()));
|
||||
break;
|
||||
}
|
||||
case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE:
|
||||
case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP:
|
||||
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE:
|
||||
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: {
|
||||
CHECK(property) << "Invalid function call!";
|
||||
wal_.WriteMarker(OperationToMarker(operation));
|
||||
wal_.WriteString(name_id_mapper_->IdToName(label.AsUint()));
|
||||
wal_.WriteString(name_id_mapper_->IdToName(property->AsUint()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
UpdateStats(timestamp);
|
||||
}
|
||||
|
||||
void WalFile::Sync() { wal_.Sync(); }
|
||||
|
||||
uint64_t WalFile::GetSize() { return wal_.GetPosition(); }
|
||||
|
||||
void WalFile::UpdateStats(uint64_t timestamp) {
|
||||
if (count_ == 0) from_timestamp_ = timestamp;
|
||||
to_timestamp_ = timestamp;
|
||||
count_ += 1;
|
||||
}
|
||||
|
||||
Durability::Durability(Config::Durability config,
|
||||
utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges,
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/constraints.hpp"
|
||||
#include "storage/v2/delta.hpp"
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/indices.hpp"
|
||||
#include "storage/v2/name_id_mapper.hpp"
|
||||
@ -47,8 +48,25 @@ enum class Marker : uint8_t {
|
||||
SECTION_METADATA = 0x23,
|
||||
SECTION_INDICES = 0x24,
|
||||
SECTION_CONSTRAINTS = 0x25,
|
||||
SECTION_DELTA = 0x26,
|
||||
SECTION_OFFSETS = 0x42,
|
||||
|
||||
DELTA_VERTEX_CREATE = 0x50,
|
||||
DELTA_VERTEX_DELETE = 0x51,
|
||||
DELTA_VERTEX_ADD_LABEL = 0x52,
|
||||
DELTA_VERTEX_REMOVE_LABEL = 0x53,
|
||||
DELTA_VERTEX_SET_PROPERTY = 0x54,
|
||||
DELTA_EDGE_CREATE = 0x55,
|
||||
DELTA_EDGE_DELETE = 0x56,
|
||||
DELTA_EDGE_SET_PROPERTY = 0x57,
|
||||
DELTA_TRANSACTION_END = 0x58,
|
||||
DELTA_LABEL_INDEX_CREATE = 0x59,
|
||||
DELTA_LABEL_INDEX_DROP = 0x5a,
|
||||
DELTA_LABEL_PROPERTY_INDEX_CREATE = 0x5b,
|
||||
DELTA_LABEL_PROPERTY_INDEX_DROP = 0x5c,
|
||||
DELTA_EXISTENCE_CONSTRAINT_CREATE = 0x5d,
|
||||
DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e,
|
||||
|
||||
VALUE_FALSE = 0x00,
|
||||
VALUE_TRUE = 0xff,
|
||||
};
|
||||
@ -56,14 +74,38 @@ enum class Marker : uint8_t {
|
||||
/// List of all available markers.
|
||||
/// IMPORTANT: Don't forget to update this list when you add a new Marker.
|
||||
static const Marker kMarkersAll[] = {
|
||||
Marker::TYPE_NULL, Marker::TYPE_BOOL,
|
||||
Marker::TYPE_INT, Marker::TYPE_DOUBLE,
|
||||
Marker::TYPE_STRING, Marker::TYPE_LIST,
|
||||
Marker::TYPE_MAP, Marker::TYPE_PROPERTY_VALUE,
|
||||
Marker::SECTION_VERTEX, Marker::SECTION_EDGE,
|
||||
Marker::SECTION_MAPPER, Marker::SECTION_METADATA,
|
||||
Marker::SECTION_INDICES, Marker::SECTION_CONSTRAINTS,
|
||||
Marker::SECTION_OFFSETS, Marker::VALUE_FALSE,
|
||||
Marker::TYPE_NULL,
|
||||
Marker::TYPE_BOOL,
|
||||
Marker::TYPE_INT,
|
||||
Marker::TYPE_DOUBLE,
|
||||
Marker::TYPE_STRING,
|
||||
Marker::TYPE_LIST,
|
||||
Marker::TYPE_MAP,
|
||||
Marker::TYPE_PROPERTY_VALUE,
|
||||
Marker::SECTION_VERTEX,
|
||||
Marker::SECTION_EDGE,
|
||||
Marker::SECTION_MAPPER,
|
||||
Marker::SECTION_METADATA,
|
||||
Marker::SECTION_INDICES,
|
||||
Marker::SECTION_CONSTRAINTS,
|
||||
Marker::SECTION_DELTA,
|
||||
Marker::SECTION_OFFSETS,
|
||||
Marker::DELTA_VERTEX_CREATE,
|
||||
Marker::DELTA_VERTEX_DELETE,
|
||||
Marker::DELTA_VERTEX_ADD_LABEL,
|
||||
Marker::DELTA_VERTEX_REMOVE_LABEL,
|
||||
Marker::DELTA_VERTEX_SET_PROPERTY,
|
||||
Marker::DELTA_EDGE_CREATE,
|
||||
Marker::DELTA_EDGE_DELETE,
|
||||
Marker::DELTA_EDGE_SET_PROPERTY,
|
||||
Marker::DELTA_TRANSACTION_END,
|
||||
Marker::DELTA_LABEL_INDEX_CREATE,
|
||||
Marker::DELTA_LABEL_INDEX_DROP,
|
||||
Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE,
|
||||
Marker::DELTA_LABEL_PROPERTY_INDEX_DROP,
|
||||
Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE,
|
||||
Marker::DELTA_EXISTENCE_CONSTRAINT_DROP,
|
||||
Marker::VALUE_FALSE,
|
||||
Marker::VALUE_TRUE,
|
||||
};
|
||||
|
||||
@ -87,6 +129,8 @@ class Encoder final {
|
||||
uint64_t GetPosition();
|
||||
void SetPosition(uint64_t position);
|
||||
|
||||
void Sync();
|
||||
|
||||
void Finalize();
|
||||
|
||||
private:
|
||||
@ -148,6 +192,69 @@ struct SnapshotInfo {
|
||||
/// @throw RecoveryFailure
|
||||
SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
|
||||
|
||||
/// Structure used to hold information about a WAL.
|
||||
struct WalInfo {
|
||||
uint64_t offset_metadata;
|
||||
uint64_t offset_deltas;
|
||||
|
||||
std::string uuid;
|
||||
uint64_t seq_num;
|
||||
uint64_t from_timestamp;
|
||||
uint64_t to_timestamp;
|
||||
uint64_t num_deltas;
|
||||
};
|
||||
|
||||
/// Function used to read information about the WAL file.
|
||||
/// @throw RecoveryFailure
|
||||
WalInfo ReadWalInfo(const std::filesystem::path &path);
|
||||
|
||||
/// Enum used to indicate a global database operation that isn't transactional.
|
||||
enum class StorageGlobalOperation {
|
||||
LABEL_INDEX_CREATE,
|
||||
LABEL_INDEX_DROP,
|
||||
LABEL_PROPERTY_INDEX_CREATE,
|
||||
LABEL_PROPERTY_INDEX_DROP,
|
||||
EXISTENCE_CONSTRAINT_CREATE,
|
||||
EXISTENCE_CONSTRAINT_DROP,
|
||||
};
|
||||
|
||||
/// WalFile class used to append deltas and operations to the WAL file.
|
||||
class WalFile {
|
||||
public:
|
||||
WalFile(const std::filesystem::path &wal_directory, const std::string &uuid,
|
||||
Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num);
|
||||
|
||||
WalFile(const WalFile &) = delete;
|
||||
WalFile(WalFile &&) = delete;
|
||||
WalFile &operator=(const WalFile &) = delete;
|
||||
WalFile &operator=(WalFile &&) = delete;
|
||||
|
||||
~WalFile();
|
||||
|
||||
void AppendDelta(const Delta &delta, Vertex *vertex, uint64_t timestamp);
|
||||
void AppendDelta(const Delta &delta, Edge *edge, uint64_t timestamp);
|
||||
|
||||
void AppendTransactionEnd(uint64_t timestamp);
|
||||
|
||||
void AppendOperation(StorageGlobalOperation operation, LabelId label,
|
||||
std::optional<PropertyId> property, uint64_t timestamp);
|
||||
|
||||
void Sync();
|
||||
|
||||
uint64_t GetSize();
|
||||
|
||||
private:
|
||||
void UpdateStats(uint64_t timestamp);
|
||||
|
||||
Config::Items items_;
|
||||
NameIdMapper *name_id_mapper_;
|
||||
Encoder wal_;
|
||||
std::filesystem::path path_;
|
||||
uint64_t from_timestamp_;
|
||||
uint64_t to_timestamp_;
|
||||
uint64_t count_;
|
||||
};
|
||||
|
||||
/// Durability class that is used to provide full durability functionality to
|
||||
/// the storage.
|
||||
class Durability final {
|
||||
|
||||
@ -28,9 +28,6 @@ namespace storage {
|
||||
// The paper implements a fully serializable storage, in our implementation we
|
||||
// only implement snapshot isolation for transactions.
|
||||
|
||||
const uint64_t kTimestampInitialId = 0;
|
||||
const uint64_t kTransactionInitialId = 1ULL << 63U;
|
||||
|
||||
/// Iterable for iterating through all vertices of a Storage.
|
||||
///
|
||||
/// An instance of this will be usually be wrapped inside VerticesIterable for
|
||||
|
||||
@ -15,6 +15,9 @@
|
||||
|
||||
namespace storage {
|
||||
|
||||
const uint64_t kTimestampInitialId = 0;
|
||||
const uint64_t kTransactionInitialId = 1ULL << 63U;
|
||||
|
||||
struct Transaction {
|
||||
Transaction(uint64_t transaction_id, uint64_t start_timestamp)
|
||||
: transaction_id(transaction_id),
|
||||
|
||||
@ -61,6 +61,13 @@ bool CopyFile(const std::filesystem::path &src,
|
||||
return std::filesystem::copy_file(src, dst, error_code);
|
||||
}
|
||||
|
||||
bool RenamePath(const std::filesystem::path &src,
|
||||
const std::filesystem::path &dst) {
|
||||
std::error_code error_code; // For exception suppression.
|
||||
std::filesystem::rename(src, dst, error_code);
|
||||
return !error_code;
|
||||
}
|
||||
|
||||
static_assert(std::is_same_v<off_t, ssize_t>, "off_t must fit into ssize_t!");
|
||||
|
||||
InputFile::~InputFile() { Close(); }
|
||||
|
||||
@ -40,6 +40,11 @@ bool DeleteFile(const std::filesystem::path &file) noexcept;
|
||||
bool CopyFile(const std::filesystem::path &src,
|
||||
const std::filesystem::path &dst) noexcept;
|
||||
|
||||
/// Renames the path from `src` to `dst`. If the `dst` contains directories that
|
||||
/// don't exist, the renaming fails. Symlinks are not followed.
|
||||
bool RenamePath(const std::filesystem::path &src,
|
||||
const std::filesystem::path &dst);
|
||||
|
||||
/// This class implements a file handler that is used to read binary files. It
|
||||
/// was developed because the C++ standard library has an awful API and makes
|
||||
/// handling of binary data extremely tedious.
|
||||
|
||||
@ -362,6 +362,9 @@ target_link_libraries(${test_prefix}storage_v2_indices mg-storage-v2)
|
||||
add_unit_test(storage_v2_name_id_mapper.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v2_name_id_mapper mg-storage-v2)
|
||||
|
||||
add_unit_test(storage_v2_wal_file.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v2_wal_file mg-storage-v2 fmt)
|
||||
|
||||
# Test LCP
|
||||
|
||||
add_custom_command(
|
||||
|
||||
@ -322,7 +322,23 @@ TEST_F(DecoderEncoderTest, PropertyValueInvalidMarker) {
|
||||
case storage::Marker::SECTION_METADATA:
|
||||
case storage::Marker::SECTION_INDICES:
|
||||
case storage::Marker::SECTION_CONSTRAINTS:
|
||||
case storage::Marker::SECTION_DELTA:
|
||||
case storage::Marker::SECTION_OFFSETS:
|
||||
case storage::Marker::DELTA_VERTEX_CREATE:
|
||||
case storage::Marker::DELTA_VERTEX_DELETE:
|
||||
case storage::Marker::DELTA_VERTEX_ADD_LABEL:
|
||||
case storage::Marker::DELTA_VERTEX_REMOVE_LABEL:
|
||||
case storage::Marker::DELTA_VERTEX_SET_PROPERTY:
|
||||
case storage::Marker::DELTA_EDGE_CREATE:
|
||||
case storage::Marker::DELTA_EDGE_DELETE:
|
||||
case storage::Marker::DELTA_EDGE_SET_PROPERTY:
|
||||
case storage::Marker::DELTA_TRANSACTION_END:
|
||||
case storage::Marker::DELTA_LABEL_INDEX_CREATE:
|
||||
case storage::Marker::DELTA_LABEL_INDEX_DROP:
|
||||
case storage::Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
|
||||
case storage::Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
|
||||
case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
|
||||
case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
|
||||
case storage::Marker::VALUE_FALSE:
|
||||
case storage::Marker::VALUE_TRUE:
|
||||
valid_marker = false;
|
||||
|
||||
506
tests/unit/storage_v2_wal_file.cpp
Normal file
506
tests/unit/storage_v2_wal_file.cpp
Normal file
@ -0,0 +1,506 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <string_view>
|
||||
|
||||
#include "storage/v2/durability.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/name_id_mapper.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
// This class mimics the internals of the storage to generate the deltas.
|
||||
class DeltaGenerator final {
|
||||
public:
|
||||
class Transaction final {
|
||||
private:
|
||||
friend class DeltaGenerator;
|
||||
|
||||
explicit Transaction(DeltaGenerator *gen)
|
||||
: gen_(gen), transaction_(gen->transaction_id_++, gen->timestamp_++) {}
|
||||
|
||||
public:
|
||||
storage::Vertex *CreateVertex() {
|
||||
auto gid = storage::Gid::FromUint(gen_->vertices_count_++);
|
||||
auto delta = storage::CreateDeleteObjectDelta(&transaction_);
|
||||
auto &it = gen_->vertices_.emplace_back(gid, delta);
|
||||
delta->prev.Set(&it);
|
||||
return ⁢
|
||||
}
|
||||
|
||||
void DeleteVertex(storage::Vertex *vertex) {
|
||||
storage::CreateAndLinkDelta(&transaction_, &*vertex,
|
||||
storage::Delta::RecreateObjectTag());
|
||||
}
|
||||
|
||||
void AddLabel(storage::Vertex *vertex, const std::string &label) {
|
||||
auto label_id = storage::LabelId::FromUint(gen_->mapper_.NameToId(label));
|
||||
vertex->labels.push_back(label_id);
|
||||
storage::CreateAndLinkDelta(&transaction_, &*vertex,
|
||||
storage::Delta::RemoveLabelTag(), label_id);
|
||||
}
|
||||
|
||||
void RemoveLabel(storage::Vertex *vertex, const std::string &label) {
|
||||
auto label_id = storage::LabelId::FromUint(gen_->mapper_.NameToId(label));
|
||||
vertex->labels.erase(
|
||||
std::find(vertex->labels.begin(), vertex->labels.end(), label_id));
|
||||
storage::CreateAndLinkDelta(&transaction_, &*vertex,
|
||||
storage::Delta::AddLabelTag(), label_id);
|
||||
}
|
||||
|
||||
void SetProperty(storage::Vertex *vertex, const std::string &property,
|
||||
const storage::PropertyValue &value) {
|
||||
auto property_id =
|
||||
storage::PropertyId::FromUint(gen_->mapper_.NameToId(property));
|
||||
auto &props = vertex->properties;
|
||||
auto it = props.find(property_id);
|
||||
if (it == props.end()) {
|
||||
storage::CreateAndLinkDelta(&transaction_, &*vertex,
|
||||
storage::Delta::SetPropertyTag(),
|
||||
property_id, storage::PropertyValue());
|
||||
if (!value.IsNull()) {
|
||||
props.emplace(property_id, value);
|
||||
}
|
||||
} else {
|
||||
storage::CreateAndLinkDelta(&transaction_, &*vertex,
|
||||
storage::Delta::SetPropertyTag(),
|
||||
property_id, it->second);
|
||||
if (!value.IsNull()) {
|
||||
it->second = value;
|
||||
} else {
|
||||
props.erase(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Finalize(bool append_transaction_end = true) {
|
||||
auto commit_timestamp = gen_->timestamp_++;
|
||||
for (const auto &delta : transaction_.deltas) {
|
||||
auto owner = delta.prev.Get();
|
||||
while (owner.type == storage::PreviousPtr::Type::DELTA) {
|
||||
owner = owner.delta->prev.Get();
|
||||
}
|
||||
if (owner.type == storage::PreviousPtr::Type::VERTEX) {
|
||||
gen_->wal_file_.AppendDelta(delta, owner.vertex, commit_timestamp);
|
||||
} else if (owner.type == storage::PreviousPtr::Type::EDGE) {
|
||||
gen_->wal_file_.AppendDelta(delta, owner.edge, commit_timestamp);
|
||||
} else {
|
||||
LOG(FATAL) << "Invalid delta owner!";
|
||||
}
|
||||
}
|
||||
if (append_transaction_end) {
|
||||
gen_->wal_file_.AppendTransactionEnd(commit_timestamp);
|
||||
gen_->UpdateStats(commit_timestamp, transaction_.deltas.size() + 1);
|
||||
} else {
|
||||
gen_->valid_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
DeltaGenerator *gen_;
|
||||
storage::Transaction transaction_;
|
||||
};
|
||||
|
||||
DeltaGenerator(const std::filesystem::path &data_directory,
|
||||
bool properties_on_edges, uint64_t seq_num)
|
||||
: uuid_(utils::GenerateUUID()),
|
||||
seq_num_(seq_num),
|
||||
wal_file_(data_directory, uuid_,
|
||||
{.properties_on_edges = properties_on_edges}, &mapper_,
|
||||
seq_num) {}
|
||||
|
||||
Transaction CreateTransaction() { return Transaction(this); }
|
||||
|
||||
void ResetTransactionIds() {
|
||||
transaction_id_ = storage::kTransactionInitialId;
|
||||
timestamp_ = storage::kTimestampInitialId;
|
||||
valid_ = false;
|
||||
}
|
||||
|
||||
void AppendOperation(storage::StorageGlobalOperation operation,
|
||||
const std::string &label,
|
||||
std::optional<std::string> property = std::nullopt) {
|
||||
auto label_id = storage::LabelId::FromUint(mapper_.NameToId(label));
|
||||
std::optional<storage::PropertyId> property_id;
|
||||
if (property) {
|
||||
property_id = storage::PropertyId::FromUint(mapper_.NameToId(*property));
|
||||
}
|
||||
wal_file_.AppendOperation(operation, label_id, property_id, timestamp_);
|
||||
UpdateStats(timestamp_, 1);
|
||||
}
|
||||
|
||||
uint64_t GetPosition() { return wal_file_.GetSize(); }
|
||||
|
||||
storage::WalInfo GetInfo() {
|
||||
return {.offset_metadata = 0,
|
||||
.offset_deltas = 0,
|
||||
.uuid = uuid_,
|
||||
.seq_num = seq_num_,
|
||||
.from_timestamp = tx_from_,
|
||||
.to_timestamp = tx_to_,
|
||||
.num_deltas = deltas_count_};
|
||||
}
|
||||
|
||||
private:
|
||||
void UpdateStats(uint64_t timestamp, uint64_t count) {
|
||||
if (!valid_) return;
|
||||
if (deltas_count_ == 0) {
|
||||
tx_from_ = timestamp;
|
||||
}
|
||||
tx_to_ = timestamp;
|
||||
deltas_count_ += count;
|
||||
}
|
||||
|
||||
std::string uuid_;
|
||||
uint64_t seq_num_;
|
||||
|
||||
uint64_t transaction_id_{storage::kTransactionInitialId};
|
||||
uint64_t timestamp_{storage::kTimestampInitialId};
|
||||
uint64_t vertices_count_{0};
|
||||
std::list<storage::Vertex> vertices_;
|
||||
storage::NameIdMapper mapper_;
|
||||
|
||||
storage::WalFile wal_file_;
|
||||
|
||||
uint64_t deltas_count_{0};
|
||||
uint64_t tx_from_{0};
|
||||
uint64_t tx_to_{0};
|
||||
uint64_t valid_{true};
|
||||
};
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define TRANSACTION(append_transaction_end, ops) \
|
||||
{ \
|
||||
auto tx = gen.CreateTransaction(); \
|
||||
ops; \
|
||||
tx.Finalize(append_transaction_end); \
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define OPERATION(op, ...) \
|
||||
gen.AppendOperation(storage::StorageGlobalOperation::op, __VA_ARGS__)
|
||||
|
||||
void AssertWalInfoEqual(const storage::WalInfo &a, const storage::WalInfo &b) {
|
||||
ASSERT_EQ(a.uuid, b.uuid);
|
||||
ASSERT_EQ(a.seq_num, b.seq_num);
|
||||
ASSERT_EQ(a.from_timestamp, b.from_timestamp);
|
||||
ASSERT_EQ(a.to_timestamp, b.to_timestamp);
|
||||
ASSERT_EQ(a.num_deltas, b.num_deltas);
|
||||
}
|
||||
|
||||
class WalFileTest : public ::testing::TestWithParam<bool> {
|
||||
public:
|
||||
WalFileTest() {}
|
||||
|
||||
void SetUp() override { Clear(); }
|
||||
|
||||
void TearDown() override { Clear(); }
|
||||
|
||||
std::vector<std::filesystem::path> GetFilesList() {
|
||||
std::vector<std::filesystem::path> ret;
|
||||
for (auto &item : std::filesystem::directory_iterator(storage_directory)) {
|
||||
ret.push_back(item.path());
|
||||
}
|
||||
std::sort(ret.begin(), ret.end());
|
||||
std::reverse(ret.begin(), ret.end());
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::filesystem::path storage_directory{
|
||||
std::filesystem::temp_directory_path() /
|
||||
"MG_test_unit_storage_v2_wal_file"};
|
||||
|
||||
private:
|
||||
void Clear() {
|
||||
if (!std::filesystem::exists(storage_directory)) return;
|
||||
std::filesystem::remove_all(storage_directory);
|
||||
}
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(EdgesWithProperties, WalFileTest,
|
||||
::testing::Values(true));
|
||||
INSTANTIATE_TEST_CASE_P(EdgesWithoutProperties, WalFileTest,
|
||||
::testing::Values(false));
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST_P(WalFileTest, EmptyFile) {
|
||||
{ DeltaGenerator gen(storage_directory, GetParam(), 5); }
|
||||
auto wal_files = GetFilesList();
|
||||
ASSERT_EQ(wal_files.size(), 0);
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define GENERATE_SIMPLE_TEST(name, ops) \
|
||||
TEST_P(WalFileTest, name) { \
|
||||
storage::WalInfo info; \
|
||||
\
|
||||
{ \
|
||||
DeltaGenerator gen(storage_directory, GetParam(), 5); \
|
||||
ops; \
|
||||
info = gen.GetInfo(); \
|
||||
} \
|
||||
\
|
||||
auto wal_files = GetFilesList(); \
|
||||
ASSERT_EQ(wal_files.size(), 1); \
|
||||
\
|
||||
if (info.num_deltas == 0) { \
|
||||
ASSERT_THROW(storage::ReadWalInfo(wal_files.front()), \
|
||||
storage::RecoveryFailure); \
|
||||
} else { \
|
||||
AssertWalInfoEqual(info, storage::ReadWalInfo(wal_files.front())); \
|
||||
} \
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionWithEnd,
|
||||
{ TRANSACTION(true, { tx.CreateVertex(); }); });
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionWithoutEnd,
|
||||
{ TRANSACTION(false, { tx.CreateVertex(); }); });
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(OperationSingle,
|
||||
{ OPERATION(LABEL_INDEX_CREATE, "hello"); });
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsEnd00, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsEnd01, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsEnd10, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsEnd11, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation_00, {
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation_01, {
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation_10, {
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation_11, {
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation0_0, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation0_1, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation1_0, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation1_1, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation00_, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation01_, {
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation10_, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(false, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(TransactionsWithOperation11_, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(AllTransactionOperationsWithEnd, {
|
||||
TRANSACTION(true, {
|
||||
auto vertex1 = tx.CreateVertex();
|
||||
auto vertex2 = tx.CreateVertex();
|
||||
tx.AddLabel(vertex1, "test");
|
||||
tx.AddLabel(vertex2, "hello");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare"));
|
||||
tx.RemoveLabel(vertex1, "test");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue(123));
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue());
|
||||
tx.DeleteVertex(vertex1);
|
||||
});
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(AllTransactionOperationsWithoutEnd, {
|
||||
TRANSACTION(false, {
|
||||
auto vertex1 = tx.CreateVertex();
|
||||
auto vertex2 = tx.CreateVertex();
|
||||
tx.AddLabel(vertex1, "test");
|
||||
tx.AddLabel(vertex2, "hello");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare"));
|
||||
tx.RemoveLabel(vertex1, "test");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue(123));
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue());
|
||||
tx.DeleteVertex(vertex1);
|
||||
});
|
||||
});
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(AllGlobalOperations, {
|
||||
OPERATION(LABEL_INDEX_CREATE, "hello");
|
||||
OPERATION(LABEL_INDEX_DROP, "hello");
|
||||
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world");
|
||||
OPERATION(LABEL_PROPERTY_INDEX_DROP, "hello", "world");
|
||||
OPERATION(EXISTENCE_CONSTRAINT_CREATE, "hello", "world");
|
||||
OPERATION(EXISTENCE_CONSTRAINT_DROP, "hello", "world");
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
GENERATE_SIMPLE_TEST(InvalidTransactionOrdering, {
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
gen.ResetTransactionIds();
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST_P(WalFileTest, InvalidMarker) {
|
||||
storage::WalInfo info;
|
||||
|
||||
{
|
||||
DeltaGenerator gen(storage_directory, GetParam(), 5);
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
info = gen.GetInfo();
|
||||
}
|
||||
|
||||
auto wal_files = GetFilesList();
|
||||
ASSERT_EQ(wal_files.size(), 1);
|
||||
const auto &wal_file = wal_files.front();
|
||||
|
||||
auto final_info = storage::ReadWalInfo(wal_file);
|
||||
AssertWalInfoEqual(info, final_info);
|
||||
|
||||
size_t i = 0;
|
||||
for (auto marker : storage::kMarkersAll) {
|
||||
if (marker == storage::Marker::SECTION_DELTA) continue;
|
||||
auto current_file = storage_directory / fmt::format("temporary_{}", i);
|
||||
ASSERT_TRUE(std::filesystem::copy_file(wal_file, current_file));
|
||||
utils::OutputFile file;
|
||||
file.Open(current_file, utils::OutputFile::Mode::OVERWRITE_EXISTING);
|
||||
file.SetPosition(utils::OutputFile::Position::SET,
|
||||
final_info.offset_deltas);
|
||||
auto value = static_cast<uint8_t>(marker);
|
||||
file.Write(&value, sizeof(value));
|
||||
file.Sync();
|
||||
file.Close();
|
||||
ASSERT_THROW(storage::ReadWalInfo(current_file), storage::RecoveryFailure);
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST_P(WalFileTest, PartialData) {
|
||||
std::vector<std::pair<uint64_t, storage::WalInfo>> infos;
|
||||
|
||||
{
|
||||
DeltaGenerator gen(storage_directory, GetParam(), 5);
|
||||
TRANSACTION(true, { tx.CreateVertex(); });
|
||||
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
|
||||
TRANSACTION(true, {
|
||||
auto vertex = tx.CreateVertex();
|
||||
tx.AddLabel(vertex, "hello");
|
||||
});
|
||||
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
|
||||
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world");
|
||||
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
|
||||
TRANSACTION(true, {
|
||||
auto vertex1 = tx.CreateVertex();
|
||||
auto vertex2 = tx.CreateVertex();
|
||||
tx.AddLabel(vertex1, "test");
|
||||
tx.AddLabel(vertex2, "hello");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue("nandare"));
|
||||
tx.RemoveLabel(vertex1, "test");
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue(123));
|
||||
tx.SetProperty(vertex2, "hello", storage::PropertyValue());
|
||||
tx.DeleteVertex(vertex1);
|
||||
});
|
||||
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
|
||||
}
|
||||
|
||||
auto wal_files = GetFilesList();
|
||||
ASSERT_EQ(wal_files.size(), 1);
|
||||
const auto &wal_file = wal_files.front();
|
||||
|
||||
AssertWalInfoEqual(infos.back().second, storage::ReadWalInfo(wal_file));
|
||||
|
||||
auto current_file = storage_directory / "temporary";
|
||||
utils::InputFile infile;
|
||||
infile.Open(wal_file);
|
||||
|
||||
uint64_t pos = 0;
|
||||
for (size_t i = 0; i < infile.GetSize(); ++i) {
|
||||
if (i < infos.front().first) {
|
||||
ASSERT_THROW(storage::ReadWalInfo(current_file),
|
||||
storage::RecoveryFailure);
|
||||
} else {
|
||||
if (i >= infos[pos + 1].first) ++pos;
|
||||
AssertWalInfoEqual(infos[pos].second, storage::ReadWalInfo(current_file));
|
||||
}
|
||||
{
|
||||
utils::OutputFile outfile;
|
||||
outfile.Open(current_file, utils::OutputFile::Mode::APPEND_TO_EXISTING);
|
||||
uint8_t value;
|
||||
ASSERT_TRUE(infile.Read(&value, sizeof(value)));
|
||||
outfile.Write(&value, sizeof(value));
|
||||
outfile.Sync();
|
||||
outfile.Close();
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(pos, infos.size() - 2);
|
||||
AssertWalInfoEqual(infos[infos.size() - 1].second,
|
||||
storage::ReadWalInfo(current_file));
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user