tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement durability functionality for unique constraints

Browse Source 
Summary:
This diff contains a necessary functionality to save and restore unique
constraint operations. The previous snapshot/WAL version is backward
compatible. Integration tests for migration from older snapshot and WAL
versions are also included.

Reviewers: mferencevic

Reviewed By: mferencevic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D2680
This commit is contained in:
Tonko Sabolcec 2020-03-06 13:19:08 +01:00
parent 8363991575
commit 6f83fff171
41 changed files with 658 additions and 60 deletions

198
src/storage/v2/durability.cpp
View File

@ -323,6 +323,8 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
case Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
case Marker::VALUE_FALSE:
case Marker::VALUE_TRUE:
return std::nullopt;
@ -416,6 +418,8 @@ bool Decoder::SkipPropertyValue() {
case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
case Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
case Marker::VALUE_FALSE:
case Marker::VALUE_TRUE:
return false;
@ -438,7 +442,10 @@ namespace {
// The current version of snapshot and WAL encoding / decoding.
// IMPORTANT: Please bump this version for every snapshot and/or WAL format
// change!!!
const uint64_t kVersion{12};
const uint64_t kVersion{13};
const uint64_t kOldestSupportedVersion{12};
const uint64_t kUniqueConstraintVersion{13};
// Snapshot format:
//
@ -484,6 +491,9 @@ const uint64_t kVersion{12};
// * existence constraints
// * label
// * property
// * unique constraints (from version 13)
// * label
// * properties
//
// 8) Name to ID mapper data
// * id to name mappings
@ -542,6 +552,9 @@ const uint64_t kVersion{12};
// existence constraint create, existence constraint drop
// * label name
// * property name
// * unique constraint create, unique constraint drop
// * label name
// * property names
// This is the prefix used for Snapshot and WAL filenames. It is a timestamp
// format that equals to: YYYYmmddHHMMSSffffff
@ -583,6 +596,10 @@ Marker OperationToMarker(StorageGlobalOperation operation) {
return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE;
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP:
return Marker::DELTA_EXISTENCE_CONSTRAINT_DROP;
case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE:
return Marker::DELTA_UNIQUE_CONSTRAINT_CREATE;
case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP:
return Marker::DELTA_UNIQUE_CONSTRAINT_DROP;
}
}
@ -647,6 +664,10 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
return WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE;
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
return WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP;
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
return WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE;
case Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
return WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP;
case Marker::TYPE_NULL:
case Marker::TYPE_BOOL:
@ -697,6 +718,8 @@ bool IsWalDeltaDataTypeTransactionEnd(WalDeltaData::Type type) {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
return true;
}
}
@ -805,6 +828,29 @@ WalDeltaData ReadSkipWalDeltaData(Decoder *wal) {
}
break;
}
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
if constexpr (read_data) {
auto label = wal->ReadString();
if (!label) throw RecoveryFailure("Invalid WAL data!");
delta.operation_label_properties.label = std::move(*label);
auto properties_count = wal->ReadUint();
if (!properties_count) throw RecoveryFailure("Invalid WAL data!");
for (uint64_t i = 0; i < *properties_count; ++i) {
auto property = wal->ReadString();
if (!property) throw RecoveryFailure("Invalid WAL data!");
delta.operation_label_properties.properties.emplace(
std::move(*property));
}
} else {
if (!wal->SkipString()) throw RecoveryFailure("Invalid WAL data!");
auto properties_count = wal->ReadUint();
if (!properties_count) throw RecoveryFailure("Invalid WAL data!");
for (uint64_t i = 0; i < *properties_count; ++i) {
if (!wal->SkipString()) throw RecoveryFailure("Invalid WAL data!");
}
}
}
}
return delta;
@ -835,6 +881,10 @@ void RemoveRecoveredIndexConstraint(std::vector<TObj> *list, TObj obj,
throw RecoveryFailure(error_message);
}
}
bool IsVersionSupported(uint64_t version) {
return version >= kOldestSupportedVersion && version <= kVersion;
}
} // namespace
// Function used to read information about the snapshot file.
@ -844,7 +894,8 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
auto version = snapshot.Initialize(path, kSnapshotMagic);
if (!version)
throw RecoveryFailure("Couldn't read snapshot magic and/or version!");
if (*version != kVersion) throw RecoveryFailure("Invalid snapshot version!");
if (!IsVersionSupported(*version))
throw RecoveryFailure("Invalid snapshot version!");
// Prepare return value.
SnapshotInfo info;
@ -912,7 +963,8 @@ WalInfo ReadWalInfo(const std::filesystem::path &path) {
auto version = wal.Initialize(path, kWalMagic);
if (!version)
throw RecoveryFailure("Couldn't read WAL magic and/or version!");
if (*version != kVersion) throw RecoveryFailure("Invalid WAL version!");
if (!IsVersionSupported(*version))
throw RecoveryFailure("Invalid WAL version!");
// Prepare return value.
WalInfo info;
@ -1045,6 +1097,12 @@ bool operator==(const WalDeltaData &a, const WalDeltaData &b) {
b.operation_label_property.label &&
a.operation_label_property.property ==
b.operation_label_property.property;
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
return a.operation_label_properties.label ==
b.operation_label_properties.label &&
a.operation_label_properties.properties ==
b.operation_label_properties.properties;
}
}
bool operator!=(const WalDeltaData &a, const WalDeltaData &b) {
@ -1063,14 +1121,14 @@ uint64_t ReadWalDeltaHeader(Decoder *wal) {
return *timestamp;
}
// Function used to either read the current WAL delta data. The WAL delta header
// must be read before calling this function.
// Function used to read the current WAL delta data. The WAL delta header must
// be read before calling this function.
WalDeltaData ReadWalDeltaData(Decoder *wal) {
return ReadSkipWalDeltaData<true>(wal);
}
// Function used to either skip the current WAL delta data. The WAL delta header
// must be read before calling this function.
// Function used to skip the current WAL delta data. The WAL delta header must
// be read before calling this function.
WalDeltaData::Type SkipWalDeltaData(Decoder *wal) {
auto delta = ReadSkipWalDeltaData<false>(wal);
return delta.type;
@ -1245,13 +1303,14 @@ void WalFile::AppendTransactionEnd(uint64_t timestamp) {
}
void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label,
std::optional<PropertyId> property,
const std::set<PropertyId> &properties,
uint64_t timestamp) {
wal_.WriteMarker(Marker::SECTION_DELTA);
wal_.WriteUint(timestamp);
switch (operation) {
case StorageGlobalOperation::LABEL_INDEX_CREATE:
case StorageGlobalOperation::LABEL_INDEX_DROP: {
CHECK(properties.empty()) << "Invalid function call!";
wal_.WriteMarker(OperationToMarker(operation));
wal_.WriteString(name_id_mapper_->IdToName(label.AsUint()));
break;
@ -1260,10 +1319,22 @@ void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label,
case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP:
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE:
case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: {
CHECK(property) << "Invalid function call!";
CHECK(properties.size() == 1) << "Invalid function call!";
wal_.WriteMarker(OperationToMarker(operation));
wal_.WriteString(name_id_mapper_->IdToName(label.AsUint()));
wal_.WriteString(name_id_mapper_->IdToName(property->AsUint()));
wal_.WriteString(
name_id_mapper_->IdToName((*properties.begin()).AsUint()));
break;
}
case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE:
case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: {
CHECK(!properties.empty()) << "Invalid function call!";
wal_.WriteMarker(OperationToMarker(operation));
wal_.WriteString(name_id_mapper_->IdToName(label.AsUint()));
wal_.WriteUint(properties.size());
for (const auto &property : properties) {
wal_.WriteString(name_id_mapper_->IdToName(property.AsUint()));
}
break;
}
}
@ -1530,10 +1601,10 @@ void Durability::AppendToWal(const Transaction &transaction,
}
void Durability::AppendToWal(StorageGlobalOperation operation, LabelId label,
std::optional<PropertyId> property,
const std::set<PropertyId> &properties,
uint64_t final_commit_timestamp) {
if (!InitializeWalFile()) return;
wal_file_->AppendOperation(operation, label, property,
wal_file_->AppendOperation(operation, label, properties,
final_commit_timestamp);
FinalizeWalFile();
}
@ -1742,6 +1813,19 @@ void Durability::CreateSnapshot(Transaction *transaction) {
write_mapping(item.second);
}
}
// Write unique constraints.
{
auto unique = constraints_->unique_constraints.ListConstraints();
snapshot.WriteUint(unique.size());
for (const auto &item : unique) {
write_mapping(item.first);
snapshot.WriteUint(item.second.size());
for (const auto &property : item.second) {
write_mapping(property);
}
}
}
}
// Write mapper data.
@ -1873,7 +1957,9 @@ void Durability::CreateSnapshot(Transaction *transaction) {
}
std::optional<Durability::RecoveryInfo> Durability::RecoverData() {
if (!utils::DirExists(snapshot_directory_)) return std::nullopt;
if (!utils::DirExists(snapshot_directory_) &&
!utils::DirExists(wal_directory_))
return std::nullopt;
// Helper lambda used to recover all discovered indices and constraints. The
// indices and constraints must be recovered after the data recovery is done
@ -1901,23 +1987,34 @@ std::optional<Durability::RecoveryInfo> Durability::RecoverData() {
if (ret.HasError() || !ret.GetValue())
throw RecoveryFailure("The existence constraint must be created here!");
}
// Recover unique constraints.
for (const auto &item : indices_constraints.constraints.unique) {
auto ret = constraints_->unique_constraints.CreateConstraint(
item.first, item.second, vertices_->access());
if (ret.HasError() ||
ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
throw RecoveryFailure("The unique constraint must be created here!");
}
};
// Array of all discovered snapshots, ordered by name.
std::vector<std::pair<std::filesystem::path, std::string>> snapshot_files;
std::error_code error_code;
for (const auto &item :
std::filesystem::directory_iterator(snapshot_directory_, error_code)) {
if (!item.is_regular_file()) continue;
try {
auto info = ReadSnapshotInfo(item.path());
snapshot_files.emplace_back(item.path(), info.uuid);
} catch (const RecoveryFailure &) {
continue;
if (utils::DirExists(snapshot_directory_)) {
for (const auto &item :
std::filesystem::directory_iterator(snapshot_directory_, error_code)) {
if (!item.is_regular_file()) continue;
try {
auto info = ReadSnapshotInfo(item.path());
snapshot_files.emplace_back(item.path(), info.uuid);
} catch (const RecoveryFailure &) {
continue;
}
}
CHECK(!error_code) << "Couldn't recover data because an error occurred: "
<< error_code.message() << "!";
}
CHECK(!error_code) << "Couldn't recover data because an error occurred: "
<< error_code.message() << "!";
RecoveryInfo recovery_info;
RecoveredIndicesAndConstraints indices_constraints;
@ -2066,7 +2163,8 @@ Durability::RecoveredSnapshot Durability::LoadSnapshot(
auto version = snapshot.Initialize(path, kSnapshotMagic);
if (!version)
throw RecoveryFailure("Couldn't read snapshot magic and/or version!");
if (*version != kVersion) throw RecoveryFailure("Invalid snapshot version!");
if (!IsVersionSupported(*version))
throw RecoveryFailure("Invalid snapshot version!");
// Cleanup of loaded data in case of failure.
bool success = false;
@ -2448,6 +2546,29 @@ Durability::RecoveredSnapshot Durability::LoadSnapshot(
"The existence constraint already exists!");
}
}
// Recover unique constraints.
// Snapshot version should be checked since unique constraints were
// implemented in later versions of snapshot.
if (*version >= kUniqueConstraintVersion) {
auto size = snapshot.ReadUint();
if (!size) throw RecoveryFailure("Invalid snapshot data!");
for (uint64_t i = 0; i < *size; ++i) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
auto properties_count = snapshot.ReadUint();
if (!properties_count) throw RecoveryFailure("Invalid snapshot data!");
std::set<PropertyId> properties;
for (uint64_t j = 0; j < *properties_count; ++j) {
auto property = snapshot.ReadUint();
if (!property) throw RecoveryFailure("Invalid snapshot data!");
properties.insert(get_property_from_id(*property));
}
AddRecoveredIndexConstraint(&indices_constraints.constraints.unique,
{get_label_from_id(*label), properties},
"The unique constraint already exists!");
}
}
}
// Recover timestamp.
@ -2469,7 +2590,8 @@ Durability::RecoveryInfo Durability::LoadWal(
auto version = wal.Initialize(path, kWalMagic);
if (!version)
throw RecoveryFailure("Couldn't read WAL magic and/or version!");
if (*version != kVersion) throw RecoveryFailure("Invalid WAL version!");
if (!IsVersionSupported(*version))
throw RecoveryFailure("Invalid WAL version!");
// Read wal info.
auto info = ReadWalInfo(path);
@ -2744,6 +2866,32 @@ Durability::RecoveryInfo Durability::LoadWal(
"The existence constraint doesn't exist!");
break;
}
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
auto label_id = LabelId::FromUint(name_id_mapper_->NameToId(
delta.operation_label_properties.label));
std::set<PropertyId> property_ids;
for (const auto &prop : delta.operation_label_properties.properties) {
property_ids.insert(
PropertyId::FromUint(name_id_mapper_->NameToId(prop)));
}
AddRecoveredIndexConstraint(&indices_constraints->constraints.unique,
{label_id, property_ids},
"The unique constraint already exists!");
break;
}
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
auto label_id = LabelId::FromUint(name_id_mapper_->NameToId(
delta.operation_label_properties.label));
std::set<PropertyId> property_ids;
for (const auto &prop : delta.operation_label_properties.properties) {
property_ids.insert(
PropertyId::FromUint(name_id_mapper_->NameToId(prop)));
}
RemoveRecoveredIndexConstraint(
&indices_constraints->constraints.unique,
{label_id, property_ids}, "The unique constraint doesn't exist!");
break;
}
}
ret.next_timestamp = std::max(ret.next_timestamp, timestamp + 1);
++deltas_applied;

31
src/storage/v2/durability.hpp
View File

@ -73,6 +73,8 @@ enum class Marker : uint8_t {
DELTA_LABEL_PROPERTY_INDEX_DROP = 0x5c,
DELTA_EXISTENCE_CONSTRAINT_CREATE = 0x5d,
DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e,
DELTA_UNIQUE_CONSTRAINT_CREATE = 0x5f,
DELTA_UNIQUE_CONSTRAINT_DROP = 0x60,
VALUE_FALSE = 0x00,
VALUE_TRUE = 0xff,
@ -112,6 +114,8 @@ static const Marker kMarkersAll[] = {
Marker::DELTA_LABEL_PROPERTY_INDEX_DROP,
Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE,
Marker::DELTA_EXISTENCE_CONSTRAINT_DROP,
Marker::DELTA_UNIQUE_CONSTRAINT_CREATE,
Marker::DELTA_UNIQUE_CONSTRAINT_DROP,
Marker::VALUE_FALSE,
Marker::VALUE_TRUE,
};
@ -233,6 +237,8 @@ struct WalDeltaData {
LABEL_PROPERTY_INDEX_DROP,
EXISTENCE_CONSTRAINT_CREATE,
EXISTENCE_CONSTRAINT_DROP,
UNIQUE_CONSTRAINT_CREATE,
UNIQUE_CONSTRAINT_DROP,
};
Type type{Type::TRANSACTION_END};
@ -267,6 +273,11 @@ struct WalDeltaData {
std::string label;
std::string property;
} operation_label_property;
struct {
std::string label;
std::set<std::string> properties;
} operation_label_properties;
};
bool operator==(const WalDeltaData &a, const WalDeltaData &b);
@ -277,15 +288,15 @@ bool operator!=(const WalDeltaData &a, const WalDeltaData &b);
/// @throw RecoveryFailure
uint64_t ReadWalDeltaHeader(Decoder *wal);
/// Function used to either read the current WAL delta data. The function
/// returns the read delta data. The WAL delta header must be read before
/// calling this function.
/// Function used to read the current WAL delta data. The function returns the
/// read delta data. The WAL delta header must be read before calling this
/// function.
/// @throw RecoveryFailure
WalDeltaData ReadWalDeltaData(Decoder *wal);
/// Function used to either skip the current WAL delta data. The function
/// returns the skipped delta type. The WAL delta header must be read before
/// calling this function.
/// Function used to skip the current WAL delta data. The function returns the
/// skipped delta type. The WAL delta header must be read before calling this
/// function.
/// @throw RecoveryFailure
WalDeltaData::Type SkipWalDeltaData(Decoder *wal);
@ -297,6 +308,8 @@ enum class StorageGlobalOperation {
LABEL_PROPERTY_INDEX_DROP,
EXISTENCE_CONSTRAINT_CREATE,
EXISTENCE_CONSTRAINT_DROP,
UNIQUE_CONSTRAINT_CREATE,
UNIQUE_CONSTRAINT_DROP,
};
/// Structure used to track indices and constraints during recovery.
@ -308,6 +321,7 @@ struct RecoveredIndicesAndConstraints {
struct {
std::vector<std::pair<LabelId, PropertyId>> existence;
std::vector<std::pair<LabelId, std::set<PropertyId>>> unique;
} constraints;
};
@ -331,7 +345,8 @@ class WalFile {
void AppendTransactionEnd(uint64_t timestamp);
void AppendOperation(StorageGlobalOperation operation, LabelId label,
std::optional<PropertyId> property, uint64_t timestamp);
const std::set<PropertyId> &properties,
uint64_t timestamp);
void Sync();
@ -380,7 +395,7 @@ class Durability final {
uint64_t final_commit_timestamp);
void AppendToWal(StorageGlobalOperation operation, LabelId label,
std::optional<PropertyId> property,
const std::set<PropertyId> &properties,
uint64_t final_commit_timestamp);
private:

41
src/storage/v2/storage.cpp
View File

@ -990,8 +990,8 @@ bool Storage::CreateIndex(LabelId label) {
// next regular transaction after this operation. This prevents collisions of
// commit timestamps between non-transactional operations and transactional
// operations.
durability_.AppendToWal(StorageGlobalOperation::LABEL_INDEX_CREATE, label,
std::nullopt, timestamp_);
durability_.AppendToWal(StorageGlobalOperation::LABEL_INDEX_CREATE, label, {},
timestamp_);
return true;
}
@ -1003,7 +1003,7 @@ bool Storage::CreateIndex(LabelId label, PropertyId property) {
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE,
label, property, timestamp_);
label, {property}, timestamp_);
return true;
}
@ -1012,8 +1012,8 @@ bool Storage::DropIndex(LabelId label) {
if (!indices_.label_index.DropIndex(label)) return false;
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::LABEL_INDEX_DROP, label,
std::nullopt, timestamp_);
durability_.AppendToWal(StorageGlobalOperation::LABEL_INDEX_DROP, label, {},
timestamp_);
return true;
}
@ -1023,7 +1023,7 @@ bool Storage::DropIndex(LabelId label, PropertyId property) {
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP,
label, property, timestamp_);
label, {property}, timestamp_);
return true;
}
@ -1042,7 +1042,7 @@ Storage::CreateExistenceConstraint(LabelId label, PropertyId property) {
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE,
label, property, timestamp_);
label, {property}, timestamp_);
return true;
}
@ -1053,7 +1053,7 @@ bool Storage::DropExistenceConstraint(LabelId label, PropertyId property) {
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP,
label, property, timestamp_);
label, {property}, timestamp_);
return true;
}
@ -1061,16 +1061,31 @@ utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus>
Storage::CreateUniqueConstraint(LabelId label,
const std::set<PropertyId> &properties) {
std::unique_lock<utils::RWLock> storage_guard(main_lock_);
// TODO(tsabolcec): Append action to the WAL.
return constraints_.unique_constraints.CreateConstraint(label, properties,
vertices_.access());
auto ret = constraints_.unique_constraints.CreateConstraint(
label, properties, vertices_.access());
if (ret.HasError() ||
ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) {
return ret;
}
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE,
label, properties, timestamp_);
return UniqueConstraints::CreationStatus::SUCCESS;
}
UniqueConstraints::DeletionStatus Storage::DropUniqueConstraint(
LabelId label, const std::set<PropertyId> &properties) {
std::unique_lock<utils::RWLock> storage_guard(main_lock_);
// TODO(tsabolcec): Append action to the WAL.
return constraints_.unique_constraints.DropConstraint(label, properties);
auto ret = constraints_.unique_constraints.DropConstraint(label, properties);
if (ret != UniqueConstraints::DeletionStatus::SUCCESS) {
return ret;
}
// For a description why using `timestamp_` is correct, see
// `CreateIndex(LabelId label)`.
durability_.AppendToWal(StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP, label,
properties, timestamp_);
return UniqueConstraints::DeletionStatus::SUCCESS;
}
ConstraintsInfo Storage::ListAllConstraints() const {

9
tests/integration/apollo_runs.yaml
View File

@ -65,6 +65,15 @@
- ../../../build_debug/src/mg_import_csv # mg_import_csv binary
- ../../../build_debug/tests/integration/mg_import_csv/tester # tester binary
- name: integration__durability
cd: durability
commands: ./runner.py
infiles:
- runner.py # runner script
- tests # tests directory
- ../../../build_debug/memgraph # memgraph binary
- ../../../build_debug/tools/src/mg_dump # memgraph dump binary
#- name: integration__ha_basic
# cd: ha/basic
# commands: TIMEOUT=480 ./runner.py

146
tests/integration/durability/runner.py Executable file
View File

@ -0,0 +1,146 @@
#!/usr/bin/python3 -u
import argparse
import atexit
import os
import shutil
import subprocess
import sys
import tempfile
import time
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", ".."))
TESTS_DIR = os.path.join(SCRIPT_DIR, "tests")
SNAPSHOT_FILE_NAME = "snapshot.bin"
WAL_FILE_NAME = "wal.bin"
DUMP_FILE_NAME = "expected.cypher"
def wait_for_server(port, delay=0.1):
cmd = ["nc", "-z", "-w", "1", "127.0.0.1", str(port)]
while subprocess.call(cmd) != 0:
time.sleep(0.01)
time.sleep(delay)
def sorted_content(file_path):
with open(file_path, 'r') as fin:
return sorted(list(map(lambda x: x.strip(), fin.readlines())))
def list_to_string(data):
ret = "[\n"
for row in data:
ret += " " + row + "\n"
ret += "]"
return ret
def execute_test(memgraph_binary, dump_binary, test_directory, test_type):
assert test_type in ["SNAPSHOT", "WAL"], \
"Test type should be either 'SNAPSHOT' or 'WAL'."
print("\033[1;36m~~ Executing test {} ({}) ~~\033[0m"
.format(os.path.relpath(test_directory, TESTS_DIR), test_type))
working_data_directory = tempfile.TemporaryDirectory()
if test_type == "SNAPSHOT":
snapshots_dir = os.path.join(working_data_directory.name, "snapshots")
os.makedirs(snapshots_dir)
shutil.copy(os.path.join(test_directory, SNAPSHOT_FILE_NAME),
snapshots_dir)
else:
wal_dir = os.path.join(working_data_directory.name, "wal")
os.makedirs(wal_dir)
shutil.copy(os.path.join(test_directory, WAL_FILE_NAME), wal_dir)
memgraph_args = [memgraph_binary,
"--storage-recover-on-startup",
"--storage-properties-on-edges",
"--data-directory", working_data_directory.name]
# Start the memgraph binary
memgraph = subprocess.Popen(memgraph_args)
time.sleep(0.1)
assert memgraph.poll() is None, "Memgraph process died prematurely!"
wait_for_server(7687)
# Register cleanup function
@atexit.register
def cleanup():
if memgraph.poll() is None:
memgraph.terminate()
assert memgraph.wait() == 0, "Memgraph process didn't exit cleanly!"
# Execute `database dump`
dump_output_file = tempfile.NamedTemporaryFile()
dump_args = [dump_binary, "--use-ssl=false"]
subprocess.run(dump_args, stdout=dump_output_file, check=True)
# Shutdown the memgraph binary
memgraph.terminate()
assert memgraph.wait() == 0, "Memgraph process didn't exit cleanly!"
# Compare dump files
expected_dump_file = os.path.join(test_directory, DUMP_FILE_NAME)
assert os.path.exists(expected_dump_file), \
"Could not find expected dump path {}".format(expected_dump_file)
queries_got = sorted_content(dump_output_file.name)
queries_expected = sorted_content(expected_dump_file)
assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
"{}".format(list_to_string(queries_got),
list_to_string(queries_expected))
print("\033[1;32m~~ Test successful ~~\033[0m\n")
def find_test_directories(directory):
"""
Finds all test directories. Test directory is a directory two levels below
the given directory which contains files 'snapshot.bin', 'wal.bin' and
'expected.cypher'.
"""
test_dirs = []
for entry_version in os.listdir(directory):
entry_version_path = os.path.join(directory, entry_version)
if not os.path.isdir(entry_version_path):
continue
for test_dir in os.listdir(entry_version_path):
test_dir_path = os.path.join(entry_version_path, test_dir)
if not os.path.isdir(test_dir_path):
continue
snapshot_file = os.path.join(test_dir_path, SNAPSHOT_FILE_NAME)
wal_file = os.path.join(test_dir_path, WAL_FILE_NAME)
dump_file = os.path.join(test_dir_path, DUMP_FILE_NAME)
if (os.path.isfile(snapshot_file) and os.path.isfile(dump_file) and
os.path.isfile(wal_file)):
test_dirs.append(test_dir_path)
else:
raise Exception("Missing data in test directory '{}'"
.format(test_dir_path))
return test_dirs
if __name__ == "__main__":
memgraph_binary = os.path.join(PROJECT_DIR, "build", "memgraph")
if not os.path.exists(memgraph_binary):
memgraph_binary = os.path.join(PROJECT_DIR, "build_debug", "memgraph")
dump_binary = os.path.join(PROJECT_DIR, "build", "tools", "src", "mg_dump")
if not os.path.exists(dump_binary):
dump_binary = os.path.join(PROJECT_DIR, "build_debug", "tools", "src",
"mg_dump")
parser = argparse.ArgumentParser()
parser.add_argument("--memgraph", default=memgraph_binary)
parser.add_argument("--dump", default=dump_binary)
args = parser.parse_args()
test_directories = find_test_directories(TESTS_DIR)
assert len(test_directories) > 0, "No tests have been found!"
for test_directory in test_directories:
execute_test(args.memgraph, args.dump, test_directory, "SNAPSHOT")
execute_test(args.memgraph, args.dump, test_directory, "WAL")
sys.exit(0)

11
tests/integration/durability/tests/v12/test_all/expected.cypher Normal file
View File

@ -0,0 +1,11 @@
CREATE INDEX ON :label;
CREATE INDEX ON :label2(prop2);
CREATE INDEX ON :label2(prop);
CREATE CONSTRAINT ON (u:label) ASSERT EXISTS (u.ext);
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__:label2 {__mg_id__: 0, prop2: ["kaj", 2, Null, {prop4: -1.341}], prop: "joj", ext: 2});
CREATE (:__mg_vertex__:label:label2 {__mg_id__: 1, prop: "joj", ext: 2});
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_all/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_all/wal.bin Normal file
View File

Binary file not shown.

10
tests/integration/durability/tests/v12/test_constraints/expected.cypher Normal file
View File

@ -0,0 +1,10 @@
CREATE CONSTRAINT ON (u:label) ASSERT EXISTS (u.prop);
CREATE CONSTRAINT ON (u:label2) ASSERT EXISTS (u.prop1);
CREATE CONSTRAINT ON (u:labellabel) ASSERT EXISTS (u.prop);
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__:label {__mg_id__: 0, prop: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 1, prop: false});
CREATE (:__mg_vertex__:label2 {__mg_id__: 2, prop1: 1});
CREATE (:__mg_vertex__:label2 {__mg_id__: 3, prop1: 2});
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_constraints/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_constraints/wal.bin Normal file
View File

Binary file not shown.

45
tests/integration/durability/tests/v12/test_edges/expected.cypher Normal file
View File

@ -0,0 +1,45 @@
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__ {__mg_id__: 0});
CREATE (:__mg_vertex__ {__mg_id__: 1});
CREATE (:__mg_vertex__ {__mg_id__: 2});
CREATE (:__mg_vertex__ {__mg_id__: 3});
CREATE (:__mg_vertex__:label {__mg_id__: 4});
CREATE (:__mg_vertex__:label {__mg_id__: 5});
CREATE (:__mg_vertex__:lab {__mg_id__: 6});
CREATE (:__mg_vertex__:lab {__mg_id__: 7});
CREATE (:__mg_vertex__:lab {__mg_id__: 8});
CREATE (:__mg_vertex__:lab2 {__mg_id__: 9});
CREATE (:__mg_vertex__:lab2 {__mg_id__: 10});
CREATE (:__mg_vertex__:lab2 {__mg_id__: 11});
CREATE (:__mg_vertex__ {__mg_id__: 12});
CREATE (:__mg_vertex__ {__mg_id__: 13});
CREATE (:__mg_vertex__ {__mg_id__: 14});
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:link2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:link2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:link2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:link]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:link2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:link2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 13 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 11 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 12 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 11 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 13 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:link3]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 12 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 13 CREATE (u)-[:link88]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 12 CREATE (u)-[:link3]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 14 CREATE (u)-[:selfedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 11 CREATE (u)-[:selfedge2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:link3]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 12 CREATE (u)-[:selfedge2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 13 CREATE (u)-[:selfedge2]->(v);
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_edges/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_edges/wal.bin Normal file
View File

Binary file not shown.

24
tests/integration/durability/tests/v12/test_edges_with_properties/expected.cypher Normal file
View File

@ -0,0 +1,24 @@
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__ {__mg_id__: 0});
CREATE (:__mg_vertex__ {__mg_id__: 1});
CREATE (:__mg_vertex__ {__mg_id__: 2});
CREATE (:__mg_vertex__ {__mg_id__: 3});
CREATE (:__mg_vertex__ {__mg_id__: 4});
CREATE (:__mg_vertex__ {__mg_id__: 5});
CREATE (:__mg_vertex__ {__mg_id__: 6});
CREATE (:__mg_vertex__ {__mg_id__: 7});
CREATE (:__mg_vertex__ {__mg_id__: 8});
CREATE (:__mg_vertex__ {__mg_id__: 9});
CREATE (:__mg_vertex__ {__mg_id__: 10});
CREATE (:__mg_vertex__ {__mg_id__: 11});
CREATE (:__mg_vertex__ {__mg_id__: 12});
CREATE (:__mg_vertex__ {__mg_id__: 13});
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:edge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:edge {prop: 1}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:edge {prop: false}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:edge2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:edge2 {prop: -3.141}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:edgelink {prop: 1, prop2: {prop4: 9}}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:edgelink {prop: [1, Null, false, ""]}]->(v);
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_edges_with_properties/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_edges_with_properties/wal.bin Normal file
View File

Binary file not shown.

16
tests/integration/durability/tests/v12/test_indices/expected.cypher Normal file
View File

@ -0,0 +1,16 @@
CREATE INDEX ON :label2;
CREATE INDEX ON :label1;
CREATE INDEX ON :label3;
CREATE INDEX ON :label(prop);
CREATE INDEX ON :label2(prop);
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__:label {__mg_id__: 0});
CREATE (:__mg_vertex__:label {__mg_id__: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 2});
CREATE (:__mg_vertex__:label {__mg_id__: 3, prop: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 4, prop: 2});
CREATE (:__mg_vertex__:label {__mg_id__: 5, prop: 3});
CREATE (:__mg_vertex__:label2 {__mg_id__: 6, prop2: 1});
CREATE (:__mg_vertex__:label3 {__mg_id__: 7});
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_indices/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_indices/wal.bin Normal file
View File

Binary file not shown.

16
tests/integration/durability/tests/v12/test_vertices/expected.cypher Normal file
View File

@ -0,0 +1,16 @@
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__ {__mg_id__: 0});
CREATE (:__mg_vertex__:label {__mg_id__: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 2, prop: false});
CREATE (:__mg_vertex__:label {__mg_id__: 3, prop: true});
CREATE (:__mg_vertex__:label2 {__mg_id__: 4, prop: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 5, prop2: 3.141});
CREATE (:__mg_vertex__:label6 {__mg_id__: 6, prop2: -314000000, prop3: true});
CREATE (:__mg_vertex__:label1:label2:label3 {__mg_id__: 7});
CREATE (:__mg_vertex__:label {__mg_id__: 8, prop: 1, prop2: 2, prop3: "str"});
CREATE (:__mg_vertex__:label1:label2 {__mg_id__: 9, prop: {prop_nes: "kaj je"}});
CREATE (:__mg_vertex__:label {__mg_id__: 10, prop_array: [1, false, Null, "str", {prop2: 2}]});
CREATE (:__mg_vertex__:label:label3 {__mg_id__: 11, prop: {prop: [1, false], prop2: {}, prop3: "test2", prop4: "test"}});
CREATE (:__mg_vertex__ {__mg_id__: 12, prop: " \n\"\'\t\\%"});
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v12/test_vertices/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v12/test_vertices/wal.bin Normal file
View File

Binary file not shown.

16
tests/integration/durability/tests/v13/test_all/expected.cypher Normal file
View File

@ -0,0 +1,16 @@
CREATE INDEX ON :label;
CREATE INDEX ON :label2(prop2);
CREATE INDEX ON :label2(prop);
CREATE CONSTRAINT ON (u:label) ASSERT EXISTS (u.ext);
CREATE CONSTRAINT ON (u:label2) ASSERT u.prop2, u.prop IS UNIQUE;
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__:label2 {__mg_id__: 0, prop2: ["kaj", 2, Null, {prop4: -1.341}], prop: "joj", ext: 2});
CREATE (:__mg_vertex__:label:label2 {__mg_id__: 1, prop: "joj", ext: 2});
CREATE (:__mg_vertex__:label2 {__mg_id__: 2, prop2: 2, prop: 1});
CREATE (:__mg_vertex__:label2 {__mg_id__: 3, prop2: 2, prop: 2});
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:link {prop: -1, ext: [false, {k: "l"}]}]->(v);
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v13/test_all/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v13/test_all/wal.bin Normal file
View File

Binary file not shown.

6
tests/integration/durability/tests/v13/test_constraints/expected.cypher Normal file
View File

@ -0,0 +1,6 @@
CREATE CONSTRAINT ON (u:label2) ASSERT EXISTS (u.ext2);
CREATE CONSTRAINT ON (u:label) ASSERT EXISTS (u.ext);
CREATE CONSTRAINT ON (u:label2) ASSERT u.a, u.b IS UNIQUE;
CREATE CONSTRAINT ON (u:label) ASSERT u.a IS UNIQUE;
CREATE CONSTRAINT ON (u:label) ASSERT u.b IS UNIQUE;
CREATE CONSTRAINT ON (u:label) ASSERT u.c IS UNIQUE;

BIN
tests/integration/durability/tests/v13/test_constraints/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v13/test_constraints/wal.bin Normal file
View File

Binary file not shown.

58
tests/integration/durability/tests/v13/test_edges/expected.cypher Normal file
View File

@ -0,0 +1,58 @@
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__ {__mg_id__: 0});
CREATE (:__mg_vertex__ {__mg_id__: 1});
CREATE (:__mg_vertex__ {__mg_id__: 2});
CREATE (:__mg_vertex__ {__mg_id__: 3});
CREATE (:__mg_vertex__ {__mg_id__: 4});
CREATE (:__mg_vertex__ {__mg_id__: 5});
CREATE (:__mg_vertex__ {__mg_id__: 6});
CREATE (:__mg_vertex__ {__mg_id__: 7});
CREATE (:__mg_vertex__ {__mg_id__: 8});
CREATE (:__mg_vertex__ {__mg_id__: 9});
CREATE (:__mg_vertex__ {__mg_id__: 10});
CREATE (:__mg_vertex__ {__mg_id__: 11});
CREATE (:__mg_vertex__ {__mg_id__: 12});
CREATE (:__mg_vertex__ {__mg_id__: 13});
CREATE (:__mg_vertex__:label {__mg_id__: 14});
CREATE (:__mg_vertex__:label {__mg_id__: 15});
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:edge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:edge {prop: 11}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:edge {prop: true}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:edge2]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:edge2 {prop: -3.141}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:edgelink {prop: {prop: 1, prop2: {prop4: 9}}}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:edgelink {prop: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:testedge]->(v);
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:testedge]->(v);
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v13/test_edges/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v13/test_edges/wal.bin Normal file
View File

Binary file not shown.

4
tests/integration/durability/tests/v13/test_indices/expected.cypher Normal file
View File

@ -0,0 +1,4 @@
CREATE INDEX ON :label2;
CREATE INDEX ON :label(prop);
CREATE INDEX ON :label(prop2);
CREATE INDEX ON :label2(prop2);

BIN
tests/integration/durability/tests/v13/test_indices/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v13/test_indices/wal.bin Normal file
View File

Binary file not shown.

16
tests/integration/durability/tests/v13/test_vertices/expected.cypher Normal file
View File

@ -0,0 +1,16 @@
CREATE INDEX ON :__mg_vertex__(__mg_id__);
CREATE (:__mg_vertex__ {__mg_id__: 0});
CREATE (:__mg_vertex__:label {__mg_id__: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 2, prop: false});
CREATE (:__mg_vertex__:label {__mg_id__: 3, prop: true});
CREATE (:__mg_vertex__:label2 {__mg_id__: 4, prop: 1});
CREATE (:__mg_vertex__:label {__mg_id__: 5, prop2: 3.141});
CREATE (:__mg_vertex__:label6 {__mg_id__: 6, prop2: -314000000, prop3: true});
CREATE (:__mg_vertex__:label1:label2:label3 {__mg_id__: 7});
CREATE (:__mg_vertex__:label {__mg_id__: 8, prop: 1, prop2: 2, prop3: "str"});
CREATE (:__mg_vertex__:label1:label2 {__mg_id__: 9, prop: {prop_nes: "kaj je"}});
CREATE (:__mg_vertex__:label {__mg_id__: 10, prop_array: [1, false, Null, "str", {prop2: 2}]});
CREATE (:__mg_vertex__:label:label3 {__mg_id__: 11, prop: {prop: [1, false], prop2: {}, prop3: "test2", prop4: "test"}});
CREATE (:__mg_vertex__ {__mg_id__: 12, prop: " \n\"\'\t\\%"});
DROP INDEX ON :__mg_vertex__(__mg_id__);
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

BIN
tests/integration/durability/tests/v13/test_vertices/snapshot.bin Normal file
View File

Binary file not shown.

BIN
tests/integration/durability/tests/v13/test_vertices/wal.bin Normal file
View File

Binary file not shown.

2
tests/unit/storage_v2_decoder_encoder.cpp
View File

@ -339,6 +339,8 @@ TEST_F(DecoderEncoderTest, PropertyValueInvalidMarker) {
case storage::Marker::DELTA_LABEL_PROPERTY_INDEX_DROP:
case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE:
case storage::Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case storage::Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
case storage::Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
case storage::Marker::VALUE_FALSE:
case storage::Marker::VALUE_TRUE:
valid_marker = false;

36
tests/unit/storage_v2_durability.cpp
View File

@ -65,6 +65,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
auto label_indexed = store->NameToLabel("base_indexed");
auto label_unindexed = store->NameToLabel("base_unindexed");
auto property_id = store->NameToProperty("id");
auto property_extra = store->NameToProperty("extra");
auto et1 = store->NameToEdgeType("base_et1");
auto et2 = store->NameToEdgeType("base_et2");
@ -78,6 +79,12 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
ASSERT_FALSE(store->CreateExistenceConstraint(label_unindexed, property_id)
.HasError());
// Create unique constraint.
ASSERT_FALSE(store
->CreateUniqueConstraint(label_unindexed,
{property_id, property_extra})
.HasError());
// Create vertices.
for (uint64_t i = 0; i < kNumBaseVertices; ++i) {
auto acc = store->Access();
@ -147,6 +154,10 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
ASSERT_FALSE(store->CreateExistenceConstraint(label_unused, property_count)
.HasError());
// Create unique constraint.
ASSERT_FALSE(store->CreateUniqueConstraint(label_unused, {property_count})
.HasError());
// Storage accessor.
std::optional<storage::Storage::Accessor> acc;
if (single_transaction) acc.emplace(store->Access());
@ -199,6 +210,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
auto base_label_indexed = store->NameToLabel("base_indexed");
auto base_label_unindexed = store->NameToLabel("base_unindexed");
auto property_id = store->NameToProperty("id");
auto property_extra = store->NameToProperty("extra");
auto et1 = store->NameToEdgeType("base_et1");
auto et2 = store->NameToEdgeType("base_et2");
@ -247,11 +259,17 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
case DatasetType::ONLY_BASE:
ASSERT_THAT(info.existence, UnorderedElementsAre(std::make_pair(
base_label_unindexed, property_id)));
ASSERT_THAT(info.unique, UnorderedElementsAre(std::make_pair(
base_label_unindexed,
std::set{property_id, property_extra})));
break;
case DatasetType::ONLY_EXTENDED:
ASSERT_THAT(info.existence,
UnorderedElementsAre(std::make_pair(extended_label_unused,
property_count)));
ASSERT_THAT(info.unique,
UnorderedElementsAre(std::make_pair(
extended_label_unused, std::set{property_count})));
break;
case DatasetType::ONLY_BASE_WITH_EXTENDED_INDICES_AND_CONSTRAINTS:
case DatasetType::ONLY_EXTENDED_WITH_BASE_INDICES_AND_CONSTRAINTS:
@ -261,6 +279,12 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
UnorderedElementsAre(
std::make_pair(base_label_unindexed, property_id),
std::make_pair(extended_label_unused, property_count)));
ASSERT_THAT(info.unique,
UnorderedElementsAre(
std::make_pair(base_label_unindexed,
std::set{property_id, property_extra}),
std::make_pair(extended_label_unused,
std::set{property_count})));
break;
}
}
@ -1406,6 +1430,7 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) {
ASSERT_EQ(indices.label_property.size(), 0);
auto constraints = store.ListAllConstraints();
ASSERT_EQ(constraints.existence.size(), 0);
ASSERT_EQ(constraints.unique.size(), 0);
auto acc = store.Access();
{
auto v1 = acc.FindVertex(gid_v1, storage::View::OLD);
@ -1509,17 +1534,21 @@ TEST_P(DurabilityTest, WalCreateAndRemoveEverything) {
CreateBaseDataset(&store, GetParam());
CreateExtendedDataset(&store);
auto indices = store.ListAllIndices();
for (auto index : indices.label) {
for (const auto &index : indices.label) {
ASSERT_TRUE(store.DropIndex(index));
}
for (auto index : indices.label_property) {
for (const auto &index : indices.label_property) {
ASSERT_TRUE(store.DropIndex(index.first, index.second));
}
auto constraints = store.ListAllConstraints();
for (auto constraint : constraints.existence) {
for (const auto &constraint : constraints.existence) {
ASSERT_TRUE(
store.DropExistenceConstraint(constraint.first, constraint.second));
}
for (const auto &constraint : constraints.unique) {
ASSERT_EQ(store.DropUniqueConstraint(constraint.first, constraint.second),
storage::UniqueConstraints::DeletionStatus::SUCCESS);
}
auto acc = store.Access();
for (auto vertex : acc.Vertices(storage::View::OLD)) {
ASSERT_TRUE(acc.DetachDeleteVertex(&vertex).HasValue());
@ -1542,6 +1571,7 @@ TEST_P(DurabilityTest, WalCreateAndRemoveEverything) {
ASSERT_EQ(indices.label_property.size(), 0);
auto constraints = store.ListAllConstraints();
ASSERT_EQ(constraints.existence.size(), 0);
ASSERT_EQ(constraints.unique.size(), 0);
auto acc = store.Access();
uint64_t count = 0;
auto iterable = acc.Vertices(storage::View::OLD);

33
tests/unit/storage_v2_wal_file.cpp
View File

@ -28,6 +28,10 @@ storage::WalDeltaData::Type StorageGlobalOperationToWalDeltaDataType(
return storage::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE;
case storage::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP:
return storage::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP;
case storage::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE:
return storage::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE;
case storage::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP:
return storage::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP;
}
}
@ -207,13 +211,14 @@ class DeltaGenerator final {
void AppendOperation(storage::StorageGlobalOperation operation,
const std::string &label,
std::optional<std::string> property = std::nullopt) {
const std::set<std::string> properties = {}) {
auto label_id = storage::LabelId::FromUint(mapper_.NameToId(label));
std::optional<storage::PropertyId> property_id;
if (property) {
property_id = storage::PropertyId::FromUint(mapper_.NameToId(*property));
std::set<storage::PropertyId> property_ids;
for (const auto &property : properties) {
property_ids.insert(
storage::PropertyId::FromUint(mapper_.NameToId(property)));
}
wal_file_.AppendOperation(operation, label_id, property_id, timestamp_);
wal_file_.AppendOperation(operation, label_id, property_ids, timestamp_);
if (valid_) {
UpdateStats(timestamp_, 1);
storage::WalDeltaData data;
@ -228,7 +233,11 @@ class DeltaGenerator final {
case storage::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE:
case storage::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP:
data.operation_label_property.label = label;
data.operation_label_property.property = *property;
data.operation_label_property.property = *properties.begin();
case storage::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE:
case storage::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP:
data.operation_label_properties.label = label;
data.operation_label_properties.properties = properties;
}
data_.emplace_back(timestamp_, data);
}
@ -515,10 +524,12 @@ GENERATE_SIMPLE_TEST(AllTransactionOperationsWithoutEnd, {
GENERATE_SIMPLE_TEST(AllGlobalOperations, {
OPERATION(LABEL_INDEX_CREATE, "hello");
OPERATION(LABEL_INDEX_DROP, "hello");
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world");
OPERATION(LABEL_PROPERTY_INDEX_DROP, "hello", "world");
OPERATION(EXISTENCE_CONSTRAINT_CREATE, "hello", "world");
OPERATION(EXISTENCE_CONSTRAINT_DROP, "hello", "world");
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"});
OPERATION(LABEL_PROPERTY_INDEX_DROP, "hello", {"world"});
OPERATION(EXISTENCE_CONSTRAINT_CREATE, "hello", {"world"});
OPERATION(EXISTENCE_CONSTRAINT_DROP, "hello", {"world"});
OPERATION(UNIQUE_CONSTRAINT_CREATE, "hello", {"world", "and", "universe"});
OPERATION(UNIQUE_CONSTRAINT_DROP, "hello", {"world", "and", "universe"});
});
// NOLINTNEXTLINE(hicpp-special-member-functions)
@ -578,7 +589,7 @@ TEST_P(WalFileTest, PartialData) {
tx.AddLabel(vertex, "hello");
});
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", "world");
OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"});
infos.emplace_back(gen.GetPosition(), gen.GetInfo());
TRANSACTION(true, {
auto vertex1 = tx.CreateVertex();