tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add work on durability

Browse Source
This commit is contained in:
Ante Pušić 2024-01-29 16:50:22 +01:00
parent 3e7e0d896c
commit 857dbfb84e
12 changed files with 128 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/dbms/inmemory/replication_handlers.cpp
View File

@ -659,6 +659,22 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label));
break;
}
case WalDeltaData::Type::TEXT_INDEX_CREATE: {
const auto &info = delta.operation_text;
spdlog::trace(" Create text index {} on :{}", info.index_name, info.label);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
if (transaction->CreateTextIndex(info.index_name, storage->NameToLabel(info.label), this).HasError())
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
break;
}
case WalDeltaData::Type::TEXT_INDEX_DROP: {
const auto &info = delta.operation_text;
spdlog::trace(" Drop text index {} on :{}", info.index_name, info.label);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
if (transaction->DropTextIndex(info.index_name).HasError())
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
break;
}
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);

10
src/storage/v2/disk/storage.cpp
View File

@ -1651,6 +1651,12 @@ utils::BasicResult<StorageManipulationError, void> DiskStorage::DiskAccessor::Co
case MetadataDelta::Action::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
throw utils::NotYetImplemented("ClearIndexStats(stats) is not implemented for DiskStorage.");
} break;
case MetadataDelta::Action::TEXT_INDEX_CREATE: {
// TODO antepusic
} break;
case MetadataDelta::Action::TEXT_INDEX_DROP: {
// TODO antepusic
} break;
case MetadataDelta::Action::EXISTENCE_CONSTRAINT_CREATE: {
const auto &info = md_delta.label_property;
if (!disk_storage->durable_metadata_.PersistLabelPropertyIndexAndExistenceConstraintCreation(
@ -1899,6 +1905,8 @@ utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::DiskAccessor:
return {};
}
// TODO antepusic move text index creation here
utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::DiskAccessor::CreateIndex(LabelId label,
PropertyId property) {
MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!");
@ -1943,6 +1951,8 @@ utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::DiskAccessor:
return {};
}
// TODO antepusic move text index deletion here
utils::BasicResult<StorageExistenceConstraintDefinitionError, void>
DiskStorage::DiskAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) {
MG_ASSERT(unique_guard_.owns_lock(), "Create existence constraint requires a unique access to the storage!");

6
src/storage/v2/durability/marker.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -60,6 +60,8 @@ enum class Marker : uint8_t {
DELTA_LABEL_INDEX_STATS_CLEAR = 0x62,
DELTA_LABEL_PROPERTY_INDEX_STATS_SET = 0x63,
DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR = 0x64,
DELTA_TEXT_INDEX_CREATE = 0x65,
DELTA_TEXT_INDEX_DROP = 0x66,
VALUE_FALSE = 0x00,
VALUE_TRUE = 0xff,
@ -103,6 +105,8 @@ static const Marker kMarkersAll[] = {
Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR,
Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE,
Marker::DELTA_LABEL_PROPERTY_INDEX_DROP,
Marker::DELTA_TEXT_INDEX_CREATE,
Marker::DELTA_TEXT_INDEX_DROP,
Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE,
Marker::DELTA_EXISTENCE_CONSTRAINT_DROP,
Marker::DELTA_UNIQUE_CONSTRAINT_CREATE,

3
src/storage/v2/durability/metadata.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -43,6 +43,7 @@ struct RecoveredIndicesAndConstraints {
std::vector<std::pair<LabelId, PropertyId>> label_property;
std::vector<std::pair<LabelId, LabelIndexStats>> label_stats;
std::vector<std::pair<LabelId, std::pair<PropertyId, LabelPropertyIndexStats>>> label_property_stats;
std::vector<std::pair<std::string, LabelId>> text;
} indices;
struct ConstraintsMetadata {

6
src/storage/v2/durability/serialization.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -350,6 +350,8 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR:
case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
case Marker::DELTA_TEXT_INDEX_CREATE:
case Marker::DELTA_TEXT_INDEX_DROP:
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
@ -453,6 +455,8 @@ bool Decoder::SkipPropertyValue() {
case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR:
case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE:
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
case Marker::DELTA_TEXT_INDEX_CREATE:
case Marker::DELTA_TEXT_INDEX_DROP:
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:

4
src/storage/v2/durability/storage_global_operation.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -23,6 +23,8 @@ enum class StorageMetadataOperation {
LABEL_PROPERTY_INDEX_DROP,
LABEL_PROPERTY_INDEX_STATS_SET,
LABEL_PROPERTY_INDEX_STATS_CLEAR,
TEXT_INDEX_CREATE,
TEXT_INDEX_DROP,
EXISTENCE_CONSTRAINT_CREATE,
EXISTENCE_CONSTRAINT_DROP,
UNIQUE_CONSTRAINT_CREATE,

26
src/storage/v2/durability/wal.cpp
View File

@ -95,6 +95,10 @@ Marker OperationToMarker(StorageMetadataOperation operation) {
return Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET;
case StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR:
return Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR;
case StorageMetadataOperation::TEXT_INDEX_CREATE:
return Marker::DELTA_TEXT_INDEX_CREATE;
case StorageMetadataOperation::TEXT_INDEX_DROP:
return Marker::DELTA_TEXT_INDEX_DROP;
case StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE:
return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE;
case StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP:
@ -168,6 +172,10 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
return WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE;
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
return WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP;
case Marker::DELTA_TEXT_INDEX_CREATE:
return WalDeltaData::Type::TEXT_INDEX_CREATE;
case Marker::DELTA_TEXT_INDEX_DROP:
return WalDeltaData::Type::TEXT_INDEX_DROP;
case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET:
return WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET;
case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR:
@ -309,6 +317,8 @@ WalDeltaData ReadSkipWalDeltaData(BaseDecoder *decoder) {
} break;
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
case WalDeltaData::Type::TEXT_INDEX_CREATE:
case WalDeltaData::Type::TEXT_INDEX_DROP:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
if constexpr (read_data) {
@ -508,6 +518,8 @@ bool operator==(const WalDeltaData &a, const WalDeltaData &b) {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
case WalDeltaData::Type::TEXT_INDEX_CREATE:
case WalDeltaData::Type::TEXT_INDEX_DROP:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
return a.operation_label_property.label == b.operation_label_property.label &&
@ -677,6 +689,8 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Storage
}
case StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE:
case StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP:
case StorageMetadataOperation::TEXT_INDEX_CREATE:
case StorageMetadataOperation::TEXT_INDEX_DROP:
case StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE:
case StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP: {
MG_ASSERT(properties.size() == 1, "Invalid function call!");
@ -934,6 +948,18 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
"The label index stats doesn't exist!");
break;
}
case WalDeltaData::Type::TEXT_INDEX_CREATE:
auto index_name = delta.operation_text.index_name;
auto label = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_text.label));
AddRecoveredIndexConstraint(&indices_constraints->indices.text, {index_name, label},
"The text index already exists!");
break;
case WalDeltaData::Type::TEXT_INDEX_DROP:
auto index_name = delta.operation_text.index_name;
auto label = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_text.label));
RemoveRecoveredIndexConstraint(&indices_constraints->indices.text, {index_name, label},
"The text index doesn't exist!");
break;
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label));
auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property));

11
src/storage/v2/durability/wal.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -67,6 +67,8 @@ struct WalDeltaData {
LABEL_PROPERTY_INDEX_DROP,
LABEL_PROPERTY_INDEX_STATS_SET,
LABEL_PROPERTY_INDEX_STATS_CLEAR,
TEXT_INDEX_CREATE,
TEXT_INDEX_DROP,
EXISTENCE_CONSTRAINT_CREATE,
EXISTENCE_CONSTRAINT_DROP,
UNIQUE_CONSTRAINT_CREATE,
@ -121,6 +123,11 @@ struct WalDeltaData {
std::string property;
std::string stats;
} operation_label_property_stats;
struct {
std::string index_name;
std::string label;
} operation_text;
};
bool operator==(const WalDeltaData &a, const WalDeltaData &b);
@ -155,6 +162,8 @@ constexpr bool IsWalDeltaDataTypeTransactionEndVersion15(const WalDeltaData::Typ
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET:
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR:
case WalDeltaData::Type::TEXT_INDEX_CREATE:
case WalDeltaData::Type::TEXT_INDEX_DROP:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:

19
src/storage/v2/inmemory/storage.cpp
View File

@ -1184,6 +1184,8 @@ utils::BasicResult<StorageIndexDefinitionError, void> InMemoryStorage::InMemoryA
return {};
}
// TODO antepusic move text index creation here
utils::BasicResult<StorageIndexDefinitionError, void> InMemoryStorage::InMemoryAccessor::DropIndex(LabelId label) {
MG_ASSERT(unique_guard_.owns_lock(), "Dropping label index requires a unique access to the storage!");
auto *in_memory = static_cast<InMemoryStorage *>(storage_);
@ -1212,6 +1214,8 @@ utils::BasicResult<StorageIndexDefinitionError, void> InMemoryStorage::InMemoryA
return {};
}
// TODO antepusic move text index deletion here
utils::BasicResult<StorageExistenceConstraintDefinitionError, void>
InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) {
MG_ASSERT(unique_guard_.owns_lock(), "Creating existence requires a unique access to the storage!");
@ -1972,6 +1976,16 @@ bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction,
AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR, info.label,
final_commit_timestamp);
} break;
case MetadataDelta::Action::TEXT_INDEX_CREATE: {
const auto &info = md_delta.text;
AppendToWalDataDefinition(durability::StorageMetadataOperation::TEXT_INDEX_CREATE, info.index_name, info.label,
final_commit_timestamp);
} break;
case MetadataDelta::Action::TEXT_INDEX_DROP: {
const auto &info = md_delta.text;
AppendToWalDataDefinition(durability::StorageMetadataOperation::TEXT_INDEX_CREATE, info.index_name, info.label,
final_commit_timestamp);
} break;
case MetadataDelta::Action::EXISTENCE_CONSTRAINT_CREATE: {
const auto &info = md_delta.label_property;
AppendToWalDataDefinition(durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE, info.label,
@ -2033,6 +2047,11 @@ void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOpera
return AppendToWalDataDefinition(operation, label, {}, {}, final_commit_timestamp);
}
void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, std::string index_name,
LabelId label, uint64_t final_commit_timestamp) {
return AppendToWalDataDefinition(operation, label, {}, {}, final_commit_timestamp);
}
utils::BasicResult<InMemoryStorage::CreateSnapshotError> InMemoryStorage::CreateSnapshot(
memgraph::replication::ReplicationRole replication_role) {
if (replication_role == memgraph::replication::ReplicationRole::REPLICA) {

17
src/storage/v2/inmemory/storage.hpp
View File

@ -367,27 +367,30 @@ class InMemoryStorage final : public Storage {
StorageInfo GetBaseInfo(bool force_directory) override;
StorageInfo GetInfo(bool force_directory, memgraph::replication::ReplicationRole replication_role) override;
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
[[nodiscard]] bool AppendToWalDataManipulation(const Transaction &transaction, uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
[[nodiscard]] bool AppendToWalDataDefinition(const Transaction &transaction, uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label,
uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label,
const std::set<PropertyId> &properties, uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, LabelIndexStats stats,
uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label,
const std::set<PropertyId> &properties, LabelPropertyIndexStats property_stats,
uint64_t final_commit_timestamp);
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label,
const std::set<PropertyId> &properties, LabelIndexStats stats,
LabelPropertyIndexStats property_stats, uint64_t final_commit_timestamp);
/// Return true in all cases except if any sync replicas have not sent confirmation.
void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, std::string index_name, LabelId label,
uint64_t final_commit_timestamp);
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});

21
src/storage/v2/metadata_delta.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -35,6 +35,8 @@ struct MetadataDelta {
LABEL_PROPERTY_INDEX_DROP,
LABEL_PROPERTY_INDEX_STATS_SET,
LABEL_PROPERTY_INDEX_STATS_CLEAR,
TEXT_INDEX_CREATE,
TEXT_INDEX_DROP,
EXISTENCE_CONSTRAINT_CREATE,
EXISTENCE_CONSTRAINT_DROP,
UNIQUE_CONSTRAINT_CREATE,
@ -57,6 +59,10 @@ struct MetadataDelta {
} label_property_index_stats_set;
static constexpr struct LabelPropertyIndexStatsClear {
} label_property_index_stats_clear;
static constexpr struct TextIndexCreate {
} text_index_create;
static constexpr struct TextIndexDrop {
} text_index_drop;
static constexpr struct ExistenceConstraintCreate {
} existence_constraint_create;
static constexpr struct ExistenceConstraintDrop {
@ -87,6 +93,12 @@ struct MetadataDelta {
MetadataDelta(LabelPropertyIndexStatsClear /*tag*/, LabelId label)
: action(Action::LABEL_PROPERTY_INDEX_STATS_CLEAR), label{label} {}
MetadataDelta(TextIndexCreate /*tag*/, std::string index_name, LabelId label)
: action(Action::TEXT_INDEX_CREATE), text{index_name, label} {}
MetadataDelta(TextIndexDrop /*tag*/, std::string index_name, LabelId label)
: action(Action::TEXT_INDEX_DROP), text{index_name, label} {}
MetadataDelta(ExistenceConstraintCreate /*tag*/, LabelId label, PropertyId property)
: action(Action::EXISTENCE_CONSTRAINT_CREATE), label_property{label, property} {}
@ -114,6 +126,8 @@ struct MetadataDelta {
case Action::LABEL_PROPERTY_INDEX_DROP:
case Action::LABEL_PROPERTY_INDEX_STATS_SET:
case Action::LABEL_PROPERTY_INDEX_STATS_CLEAR:
case Action::TEXT_INDEX_CREATE:
case Action::TEXT_INDEX_DROP:
case Action::EXISTENCE_CONSTRAINT_CREATE:
case Action::EXISTENCE_CONSTRAINT_DROP:
break;
@ -149,6 +163,11 @@ struct MetadataDelta {
PropertyId property;
LabelPropertyIndexStats stats;
} label_property_stats;
struct {
std::string index_name;
LabelId label;
} text;
};
};

3
src/storage/v2/replication/replication_client.cpp
View File

@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -316,6 +316,7 @@ void ReplicaStream::AppendOperation(durability::StorageMetadataOperation operati
const std::set<PropertyId> &properties, const LabelIndexStats &stats,
const LabelPropertyIndexStats &property_stats, uint64_t timestamp) {
replication::Encoder encoder(stream_.GetBuilder());
// TODO antepusic
EncodeOperation(&encoder, storage_->name_id_mapper_.get(), operation, label, properties, stats, property_stats,
timestamp);
}