Add epoch id and refactor replication client/server (#51)

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Change Log
 
+## Future
+
+### Breaking Changes
+
+* Added extra information in durability files to support replication making
+  it incompatible with the durability files generated by older versions of
+  Memgraph. Even though the replication is an Enterprise feature, the files
+  are compatible with the Community version.
+
 ## v1.2.0
 
 ### Breaking Changes

src/storage/v2/CMakeLists.txt

@@ -19,7 +19,8 @@ if(MG_ENTERPRISE)
 
     set(storage_v2_src_files
         ${storage_v2_src_files}
-        replication/replication.cpp
+        replication/replication_client.cpp
+        replication/replication_server.cpp
         replication/serialization.cpp
         replication/slk.cpp
         ${lcp_storage_cpp_files})

src/storage/v2/durability/durability.cpp

@@ -92,7 +92,7 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(
           (!current_seq_num || info.seq_num < current_seq_num))
         wal_files.emplace_back(info.seq_num, info.from_timestamp,
                                info.to_timestamp, std::move(info.uuid),
-                               item.path());
+                               std::move(info.epoch_id), item.path());
     } catch (const RecoveryFailure &e) {
       DLOG(WARNING) << "Failed to read " << item.path();
       continue;
@@ -146,6 +146,8 @@ void RecoverIndicesAndConstraints(
 std::optional<RecoveryInfo> RecoverData(
     const std::filesystem::path &snapshot_directory,
     const std::filesystem::path &wal_directory, std::string *uuid,
+    std::string *epoch_id,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
     utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
     std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
     Indices *indices, Constraints *constraints, Config::Items items,
@@ -174,8 +176,8 @@ std::optional<RecoveryInfo> RecoverData(
       }
       LOG(INFO) << "Starting snapshot recovery from " << path;
       try {
-        recovered_snapshot = LoadSnapshot(path, vertices, edges, name_id_mapper,
-                                          edge_count, items);
+        recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history,
+                                          name_id_mapper, edge_count, items);
         LOG(INFO) << "Snapshot recovery successful!";
         break;
       } catch (const RecoveryFailure &e) {
@@ -191,6 +193,8 @@ std::optional<RecoveryInfo> RecoverData(
     recovery_info = recovered_snapshot->recovery_info;
     indices_constraints = std::move(recovered_snapshot->indices_constraints);
     snapshot_timestamp = recovered_snapshot->snapshot_info.start_timestamp;
+    *epoch_id = std::move(recovered_snapshot->snapshot_info.epoch_id);
+
     if (!utils::DirExists(wal_directory)) {
       RecoverIndicesAndConstraints(indices_constraints, indices, constraints,
                                    vertices);
@@ -199,14 +203,29 @@ std::optional<RecoveryInfo> RecoverData(
   } else {
     std::error_code error_code;
     if (!utils::DirExists(wal_directory)) return std::nullopt;
-    // Array of all discovered WAL files, ordered by name.
-    std::vector<std::pair<std::filesystem::path, std::string>> wal_files;
+    // We use this smaller struct that contains only a subset of information
+    // necessary for the rest of the recovery function.
+    // Also, the struct is sorted primarily on the path it contains.
+    struct WalFileInfo {
+      explicit WalFileInfo(std::filesystem::path path, std::string uuid,
+                           std::string epoch_id)
+          : path(std::move(path)),
+            uuid(std::move(uuid)),
+            epoch_id(std::move(epoch_id)) {}
+      std::filesystem::path path;
+      std::string uuid;
+      std::string epoch_id;
+
+      auto operator<=>(const WalFileInfo &) const = default;
+    };
+    std::vector<WalFileInfo> wal_files;
     for (const auto &item :
          std::filesystem::directory_iterator(wal_directory, error_code)) {
       if (!item.is_regular_file()) continue;
       try {
         auto info = ReadWalInfo(item.path());
-        wal_files.emplace_back(item.path(), info.uuid);
+        wal_files.emplace_back(item.path(), std::move(info.uuid),
+                               std::move(info.epoch_id));
       } catch (const RecoveryFailure &e) {
         continue;
       }
@@ -216,7 +235,9 @@ std::optional<RecoveryInfo> RecoverData(
     if (wal_files.empty()) return std::nullopt;
     std::sort(wal_files.begin(), wal_files.end());
     // UUID used for durability is the UUID of the last WAL file.
-    *uuid = wal_files.back().second;
+    // Same for the epoch id.
+    *uuid = std::move(wal_files.back().uuid);
+    *epoch_id = std::move(wal_files.back().epoch_id);
   }
 
   auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
@@ -238,9 +259,8 @@ std::optional<RecoveryInfo> RecoverData(
 
   if (!wal_files.empty()) {
     {
-      const auto &[seq_num, from_timestamp, to_timestamp, _, path] =
-          wal_files[0];
-      if (seq_num != 0) {
+      const auto &first_wal = wal_files[0];
+      if (first_wal.seq_num != 0) {
         // We don't have all WAL files. We need to see whether we need them all.
         if (!snapshot_timestamp) {
           // We didn't recover from a snapshot and we must have all WAL files
@@ -248,7 +268,7 @@ std::optional<RecoveryInfo> RecoverData(
           // data from them.
           LOG(FATAL) << "There are missing prefix WAL files and data can't be "
                         "recovered without them!";
-        } else if (to_timestamp >= *snapshot_timestamp) {
+        } else if (first_wal.to_timestamp >= *snapshot_timestamp) {
           // We recovered from a snapshot and we must have at least one WAL file
           // whose all deltas were created before the snapshot in order to
           // verify that nothing is missing from the beginning of the WAL chain.
@@ -259,28 +279,42 @@ std::optional<RecoveryInfo> RecoverData(
     }
     std::optional<uint64_t> previous_seq_num;
     auto last_loaded_timestamp = snapshot_timestamp;
-    for (const auto &[seq_num, from_timestamp, to_timestamp, _, path] :
-         wal_files) {
-      if (previous_seq_num && *previous_seq_num + 1 != seq_num &&
-          *previous_seq_num != seq_num) {
+    for (auto &wal_file : wal_files) {
+      if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) {
         LOG(FATAL) << "You are missing a WAL file with the sequence number "
                    << *previous_seq_num + 1 << "!";
       }
-      previous_seq_num = seq_num;
+      previous_seq_num = wal_file.seq_num;
+
+      if (wal_file.epoch_id != *epoch_id) {
+        // This way we skip WALs finalized only because of role change.
+        // We can also set the last timestamp to 0 if last loaded timestamp
+        // is nullopt as this can only happen if the WAL file with seq = 0
+        // does not contain any deltas and we didn't find any snapshots.
+        if (last_loaded_timestamp) {
+          epoch_history->emplace_back(wal_file.epoch_id,
+                                      *last_loaded_timestamp);
+        }
+        *epoch_id = std::move(wal_file.epoch_id);
+      }
       try {
-        auto info = LoadWal(path, &indices_constraints, last_loaded_timestamp,
-                            vertices, edges, name_id_mapper, edge_count, items);
+        auto info =
+            LoadWal(wal_file.path, &indices_constraints, last_loaded_timestamp,
+                    vertices, edges, name_id_mapper, edge_count, items);
         recovery_info.next_vertex_id =
             std::max(recovery_info.next_vertex_id, info.next_vertex_id);
         recovery_info.next_edge_id =
             std::max(recovery_info.next_edge_id, info.next_edge_id);
         recovery_info.next_timestamp =
             std::max(recovery_info.next_timestamp, info.next_timestamp);
-        last_loaded_timestamp.emplace(recovery_info.next_timestamp - 1);
       } catch (const RecoveryFailure &e) {
-        LOG(FATAL) << "Couldn't recover WAL deltas from " << path
+        LOG(FATAL) << "Couldn't recover WAL deltas from " << wal_file.path
                    << " because of: " << e.what();
       }
+
+      if (recovery_info.next_timestamp != 0) {
+        last_loaded_timestamp.emplace(recovery_info.next_timestamp - 1);
+      }
     }
     // The sequence number needs to be recovered even though `LoadWal` didn't
     // load any deltas from that file.

src/storage/v2/durability/durability.hpp

@@ -55,17 +55,19 @@ struct WalDurabilityInfo {
   explicit WalDurabilityInfo(const uint64_t seq_num,
                              const uint64_t from_timestamp,
                              const uint64_t to_timestamp, std::string uuid,
-                             std::filesystem::path path)
+                             std::string epoch_id, std::filesystem::path path)
       : seq_num(seq_num),
         from_timestamp(from_timestamp),
         to_timestamp(to_timestamp),
         uuid(std::move(uuid)),
+        epoch_id(std::move(epoch_id)),
         path(std::move(path)) {}
 
   uint64_t seq_num;
   uint64_t from_timestamp;
   uint64_t to_timestamp;
   std::string uuid;
+  std::string epoch_id;
   std::filesystem::path path;
 
   auto operator<=>(const WalDurabilityInfo &) const = default;
@@ -100,6 +102,8 @@ void RecoverIndicesAndConstraints(
 std::optional<RecoveryInfo> RecoverData(
     const std::filesystem::path &snapshot_directory,
     const std::filesystem::path &wal_directory, std::string *uuid,
+    std::string *epoch_id,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
     utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
     std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
     Indices *indices, Constraints *constraints, Config::Items items,

src/storage/v2/durability/marker.hpp

@@ -24,6 +24,7 @@ enum class Marker : uint8_t {
   SECTION_INDICES = 0x24,
   SECTION_CONSTRAINTS = 0x25,
   SECTION_DELTA = 0x26,
+  SECTION_EPOCH_HISTORY = 0x27,
   SECTION_OFFSETS = 0x42,
 
   DELTA_VERTEX_CREATE = 0x50,
@@ -66,6 +67,7 @@ static const Marker kMarkersAll[] = {
     Marker::SECTION_INDICES,
     Marker::SECTION_CONSTRAINTS,
     Marker::SECTION_DELTA,
+    Marker::SECTION_EPOCH_HISTORY,
     Marker::SECTION_OFFSETS,
     Marker::DELTA_VERTEX_CREATE,
     Marker::DELTA_VERTEX_DELETE,

src/storage/v2/durability/serialization.cpp

@@ -317,6 +317,7 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::DELTA_VERTEX_CREATE:
     case Marker::DELTA_VERTEX_DELETE:
@@ -412,6 +413,7 @@ bool Decoder::SkipPropertyValue() {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::DELTA_VERTEX_CREATE:
     case Marker::DELTA_VERTEX_DELETE:

src/storage/v2/durability/snapshot.cpp

@@ -114,6 +114,7 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
     info.offset_indices = read_offset();
     info.offset_constraints = read_offset();
     info.offset_mapper = read_offset();
+    info.offset_epoch_history = read_offset();
     info.offset_metadata = read_offset();
   }
 
@@ -130,6 +131,10 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
     if (!maybe_uuid) throw RecoveryFailure("Invalid snapshot data!");
     info.uuid = std::move(*maybe_uuid);
 
+    auto maybe_epoch_id = snapshot.ReadString();
+    if (!maybe_epoch_id) throw RecoveryFailure("Invalid snapshot data!");
+    info.epoch_id = std::move(*maybe_epoch_id);
+
     auto maybe_timestamp = snapshot.ReadUint();
     if (!maybe_timestamp) throw RecoveryFailure("Invalid snapshot data!");
     info.start_timestamp = *maybe_timestamp;
@@ -146,12 +151,12 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
   return info;
 }
 
-RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
-                               utils::SkipList<Vertex> *vertices,
-                               utils::SkipList<Edge> *edges,
-                               NameIdMapper *name_id_mapper,
-                               std::atomic<uint64_t> *edge_count,
-                               Config::Items items) {
+RecoveredSnapshot LoadSnapshot(
+    const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
+    NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
+    Config::Items items) {
   RecoveryInfo ret;
   RecoveredIndicesAndConstraints indices_constraints;
 
@@ -567,6 +572,34 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
     }
   }
 
+  // Recover epoch history
+  {
+    if (!snapshot.SetPosition(info.offset_epoch_history))
+      throw RecoveryFailure("Couldn't read data from snapshot!");
+
+    const auto marker = snapshot.ReadMarker();
+    if (!marker || *marker != Marker::SECTION_EPOCH_HISTORY)
+      throw RecoveryFailure("Invalid snapshot data!");
+
+    const auto history_size = snapshot.ReadUint();
+    if (!history_size) {
+      throw RecoveryFailure("Invalid snapshot data!");
+    }
+
+    for (int i = 0; i < *history_size; ++i) {
+      auto maybe_epoch_id = snapshot.ReadString();
+      if (!maybe_epoch_id) {
+        throw RecoveryFailure("Invalid snapshot data!");
+      }
+      const auto maybe_last_commit_timestamp = snapshot.ReadUint();
+      if (!maybe_last_commit_timestamp) {
+        throw RecoveryFailure("Invalid snapshot data!");
+      }
+      epoch_history->emplace_back(std::move(*maybe_epoch_id),
+                                  *maybe_last_commit_timestamp);
+    }
+  }
+
   // Recover timestamp.
   ret.next_timestamp = info.start_timestamp + 1;
 
@@ -576,15 +609,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
   return {info, ret, std::move(indices_constraints)};
 }
 
-void CreateSnapshot(Transaction *transaction,
-                    const std::filesystem::path &snapshot_directory,
-                    const std::filesystem::path &wal_directory,
-                    uint64_t snapshot_retention_count,
-                    utils::SkipList<Vertex> *vertices,
-                    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
-                    Indices *indices, Constraints *constraints,
-                    Config::Items items, const std::string &uuid,
-                    utils::FileRetainer *file_retainer) {
+void CreateSnapshot(
+    Transaction *transaction, const std::filesystem::path &snapshot_directory,
+    const std::filesystem::path &wal_directory,
+    uint64_t snapshot_retention_count, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
+    Indices *indices, Constraints *constraints, Config::Items items,
+    const std::string &uuid, const std::string_view epoch_id,
+    const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
+    utils::FileRetainer *file_retainer) {
   // Ensure that the storage directory exists.
   utils::EnsureDirOrDie(snapshot_directory);
 
@@ -603,6 +636,7 @@ void CreateSnapshot(Transaction *transaction,
   uint64_t offset_constraints = 0;
   uint64_t offset_mapper = 0;
   uint64_t offset_metadata = 0;
+  uint64_t offset_epoch_history = 0;
   {
     snapshot.WriteMarker(Marker::SECTION_OFFSETS);
     offset_offsets = snapshot.GetPosition();
@@ -611,6 +645,7 @@ void CreateSnapshot(Transaction *transaction,
     snapshot.WriteUint(offset_indices);
     snapshot.WriteUint(offset_constraints);
     snapshot.WriteUint(offset_mapper);
+    snapshot.WriteUint(offset_epoch_history);
     snapshot.WriteUint(offset_metadata);
   }
 
@@ -814,11 +849,23 @@ void CreateSnapshot(Transaction *transaction,
     }
   }
 
+  // Write epoch history
+  {
+    offset_epoch_history = snapshot.GetPosition();
+    snapshot.WriteMarker(Marker::SECTION_EPOCH_HISTORY);
+    snapshot.WriteUint(epoch_history.size());
+    for (const auto &[epoch_id, last_commit_timestamp] : epoch_history) {
+      snapshot.WriteString(epoch_id);
+      snapshot.WriteUint(last_commit_timestamp);
+    }
+  }
+
   // Write metadata.
   {
     offset_metadata = snapshot.GetPosition();
     snapshot.WriteMarker(Marker::SECTION_METADATA);
     snapshot.WriteString(uuid);
+    snapshot.WriteString(epoch_id);
     snapshot.WriteUint(transaction->start_timestamp);
     snapshot.WriteUint(edges_count);
     snapshot.WriteUint(vertices_count);
@@ -832,6 +879,7 @@ void CreateSnapshot(Transaction *transaction,
     snapshot.WriteUint(offset_indices);
     snapshot.WriteUint(offset_constraints);
     snapshot.WriteUint(offset_mapper);
+    snapshot.WriteUint(offset_epoch_history);
     snapshot.WriteUint(offset_metadata);
   }
 

src/storage/v2/durability/snapshot.hpp

@@ -24,9 +24,11 @@ struct SnapshotInfo {
   uint64_t offset_indices;
   uint64_t offset_constraints;
   uint64_t offset_mapper;
+  uint64_t offset_epoch_history;
   uint64_t offset_metadata;
 
   std::string uuid;
+  std::string epoch_id;
   uint64_t start_timestamp;
   uint64_t edges_count;
   uint64_t vertices_count;
@@ -46,22 +48,22 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
 
 /// Function used to load the snapshot data into the storage.
 /// @throw RecoveryFailure
-RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
-                               utils::SkipList<Vertex> *vertices,
-                               utils::SkipList<Edge> *edges,
-                               NameIdMapper *name_id_mapper,
-                               std::atomic<uint64_t> *edge_count,
-                               Config::Items items);
+RecoveredSnapshot LoadSnapshot(
+    const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
+    NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
+    Config::Items items);
 
 /// Function used to create a snapshot using the given transaction.
-void CreateSnapshot(Transaction *transaction,
-                    const std::filesystem::path &snapshot_directory,
-                    const std::filesystem::path &wal_directory,
-                    uint64_t snapshot_retention_count,
-                    utils::SkipList<Vertex> *vertices,
-                    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
-                    Indices *indices, Constraints *constraints,
-                    Config::Items items, const std::string &uuid,
-                    utils::FileRetainer *file_retainer);
+void CreateSnapshot(
+    Transaction *transaction, const std::filesystem::path &snapshot_directory,
+    const std::filesystem::path &wal_directory,
+    uint64_t snapshot_retention_count, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
+    Indices *indices, Constraints *constraints, Config::Items items,
+    const std::string &uuid, std::string_view epoch_id,
+    const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
+    utils::FileRetainer *file_retainer);
 
 }  // namespace storage::durability

src/storage/v2/durability/version.hpp

@@ -9,9 +9,9 @@ namespace storage::durability {
 // The current version of snapshot and WAL encoding / decoding.
 // IMPORTANT: Please bump this version for every snapshot and/or WAL format
 // change!!!
-const uint64_t kVersion{13};
+const uint64_t kVersion{14};
 
-const uint64_t kOldestSupportedVersion{12};
+const uint64_t kOldestSupportedVersion{14};
 const uint64_t kUniqueConstraintVersion{13};
 
 // Magic values written to the start of a snapshot/WAL file to identify it.

src/storage/v2/durability/wal.cpp

@@ -165,6 +165,7 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::VALUE_FALSE:
     case Marker::VALUE_TRUE:
@@ -172,39 +173,6 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
   }
 }
 
-bool IsWalDeltaDataTypeTransactionEnd(WalDeltaData::Type type) {
-  switch (type) {
-    // These delta actions are all found inside transactions so they don't
-    // indicate a transaction end.
-    case WalDeltaData::Type::VERTEX_CREATE:
-    case WalDeltaData::Type::VERTEX_DELETE:
-    case WalDeltaData::Type::VERTEX_ADD_LABEL:
-    case WalDeltaData::Type::VERTEX_REMOVE_LABEL:
-    case WalDeltaData::Type::EDGE_CREATE:
-    case WalDeltaData::Type::EDGE_DELETE:
-    case WalDeltaData::Type::VERTEX_SET_PROPERTY:
-    case WalDeltaData::Type::EDGE_SET_PROPERTY:
-      return false;
-
-    // This delta explicitly indicates that a transaction is done.
-    case WalDeltaData::Type::TRANSACTION_END:
-      return true;
-
-    // These operations aren't transactional and they are encoded only using
-    // a single delta, so they each individually mark the end of their
-    // 'transaction'.
-    case WalDeltaData::Type::LABEL_INDEX_CREATE:
-    case WalDeltaData::Type::LABEL_INDEX_DROP:
-    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
-    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
-    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
-    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
-    case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
-    case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
-      return true;
-  }
-}
-
 // Function used to either read or skip the current WAL delta data. The WAL
 // delta header must be read before calling this function. If the delta data is
 // read then the data returned is valid, if the delta data is skipped then the
@@ -386,6 +354,10 @@ WalInfo ReadWalInfo(const std::filesystem::path &path) {
     if (!maybe_uuid) throw RecoveryFailure("Invalid WAL data!");
     info.uuid = std::move(*maybe_uuid);
 
+    auto maybe_epoch_id = wal.ReadString();
+    if (!maybe_epoch_id) throw RecoveryFailure("Invalid WAL data!");
+    info.epoch_id = std::move(*maybe_epoch_id);
+
     auto maybe_seq_num = wal.ReadUint();
     if (!maybe_seq_num) throw RecoveryFailure("Invalid WAL data!");
     info.seq_num = *maybe_seq_num;
@@ -970,9 +942,9 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
 }
 
 WalFile::WalFile(const std::filesystem::path &wal_directory,
-                 const std::string &uuid, Config::Items items,
-                 NameIdMapper *name_id_mapper, uint64_t seq_num,
-                 utils::FileRetainer *file_retainer)
+                 const std::string_view uuid, const std::string_view epoch_id,
+                 Config::Items items, NameIdMapper *name_id_mapper,
+                 uint64_t seq_num, utils::FileRetainer *file_retainer)
     : items_(items),
       name_id_mapper_(name_id_mapper),
       path_(wal_directory / MakeWalName()),
@@ -1000,6 +972,7 @@ WalFile::WalFile(const std::filesystem::path &wal_directory,
   offset_metadata = wal_.GetPosition();
   wal_.WriteMarker(Marker::SECTION_METADATA);
   wal_.WriteString(uuid);
+  wal_.WriteString(epoch_id);
   wal_.WriteUint(seq_num);
 
   // Write final offsets.

src/storage/v2/durability/wal.hpp

@@ -25,6 +25,7 @@ struct WalInfo {
   uint64_t offset_deltas;
 
   std::string uuid;
+  std::string epoch_id;
   uint64_t seq_num;
   uint64_t from_timestamp;
   uint64_t to_timestamp;
@@ -107,6 +108,39 @@ enum class StorageGlobalOperation {
   UNIQUE_CONSTRAINT_DROP,
 };
 
+constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) {
+  switch (type) {
+    // These delta actions are all found inside transactions so they don't
+    // indicate a transaction end.
+    case WalDeltaData::Type::VERTEX_CREATE:
+    case WalDeltaData::Type::VERTEX_DELETE:
+    case WalDeltaData::Type::VERTEX_ADD_LABEL:
+    case WalDeltaData::Type::VERTEX_REMOVE_LABEL:
+    case WalDeltaData::Type::EDGE_CREATE:
+    case WalDeltaData::Type::EDGE_DELETE:
+    case WalDeltaData::Type::VERTEX_SET_PROPERTY:
+    case WalDeltaData::Type::EDGE_SET_PROPERTY:
+      return false;
+
+    // This delta explicitly indicates that a transaction is done.
+    case WalDeltaData::Type::TRANSACTION_END:
+      return true;
+
+    // These operations aren't transactional and they are encoded only using
+    // a single delta, so they each individually mark the end of their
+    // 'transaction'.
+    case WalDeltaData::Type::LABEL_INDEX_CREATE:
+    case WalDeltaData::Type::LABEL_INDEX_DROP:
+    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
+    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
+    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
+    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
+    case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
+    case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
+      return true;
+  }
+}
+
 /// Function used to read information about the WAL file.
 /// @throw RecoveryFailure
 WalInfo ReadWalInfo(const std::filesystem::path &path);
@@ -158,8 +192,9 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
 /// WalFile class used to append deltas and operations to the WAL file.
 class WalFile {
  public:
-  WalFile(const std::filesystem::path &wal_directory, const std::string &uuid,
-          Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num,
+  WalFile(const std::filesystem::path &wal_directory, std::string_view uuid,
+          std::string_view epoch_id, Config::Items items,
+          NameIdMapper *name_id_mapper, uint64_t seq_num,
           utils::FileRetainer *file_retainer);
   WalFile(std::filesystem::path current_wal_path, Config::Items items,
           NameIdMapper *name_id_mapper, uint64_t seq_num,

src/storage/v2/replication/enums.hpp

@@ -0,0 +1,13 @@
+#pragma once
+#include <cstdint>
+
+namespace storage::replication {
+enum class ReplicationMode : std::uint8_t { SYNC, ASYNC };
+
+enum class ReplicaState : std::uint8_t {
+  READY,
+  REPLICATING,
+  RECOVERY,
+  INVALID
+};
+}  // namespace storage::replication

src/storage/v2/replication/replication_client.cpp

@@ -1,12 +1,13 @@
-#include "storage/v2/replication/replication.hpp"
+#include "storage/v2/replication/replication_client.hpp"
 
 #include <algorithm>
 #include <type_traits>
 
 #include "storage/v2/durability/durability.hpp"
+#include "storage/v2/replication/enums.hpp"
 #include "utils/file_locker.hpp"
 
-namespace storage::replication {
+namespace storage {
 
 namespace {
 template <typename>
@@ -14,63 +15,68 @@ template <typename>
 }  // namespace
 
 ////// ReplicationClient //////
-ReplicationClient::ReplicationClient(
-    std::string name, const std::atomic<uint64_t> &last_commit_timestamp,
-    NameIdMapper *name_id_mapper, Config::Items items,
-    utils::FileRetainer *file_retainer,
-    const std::filesystem::path &snapshot_directory,
-    const std::filesystem::path &wal_directory, const std::string_view uuid,
-    std::optional<durability::WalFile> *wal_file_ptr,
-    utils::SpinLock *transaction_engine_lock,
-    const io::network::Endpoint &endpoint, bool use_ssl,
-    const ReplicationMode mode)
+Storage::ReplicationClient::ReplicationClient(
+    std::string name, Storage *storage, const io::network::Endpoint &endpoint,
+    bool use_ssl, const replication::ReplicationMode mode)
     : name_(std::move(name)),
-      last_commit_timestamp_{last_commit_timestamp},
-      name_id_mapper_(name_id_mapper),
-      items_(items),
-      file_retainer_(file_retainer),
-      snapshot_directory_(snapshot_directory),
-      wal_directory_(wal_directory),
-      uuid_(uuid),
-      wal_file_ptr_(wal_file_ptr),
-      transaction_engine_lock_(transaction_engine_lock),
+      storage_(storage),
       rpc_context_(use_ssl),
       rpc_client_(endpoint, &rpc_context_),
       mode_(mode) {
   InitializeClient();
 }
 
-void ReplicationClient::InitializeClient() {
+void Storage::ReplicationClient::InitializeClient() {
   uint64_t current_commit_timestamp{kTimestampInitialId};
-  auto stream{rpc_client_.Stream<HeartbeatRpc>()};
+  auto stream{
+      rpc_client_.Stream<HeartbeatRpc>(storage_->last_commit_timestamp_)};
+  replication::Encoder encoder{stream.GetBuilder()};
+  // Write epoch id
+  {
+    // We need to lock so the epoch id isn't overwritten
+    std::unique_lock engine_guard{storage_->engine_lock_};
+    encoder.WriteString(storage_->epoch_id_);
+  }
   const auto response = stream.AwaitResponse();
+  if (!response.success) {
+    LOG(ERROR)
+        << "Replica " << name_
+        << " is ahead of this instance. The branching point is on commit "
+        << response.current_commit_timestamp;
+    return;
+  }
   current_commit_timestamp = response.current_commit_timestamp;
   DLOG(INFO) << "CURRENT TIMESTAMP: " << current_commit_timestamp;
-  DLOG(INFO) << "CURRENT MAIN TIMESTAMP: " << last_commit_timestamp_.load();
-  if (current_commit_timestamp == last_commit_timestamp_.load()) {
+  DLOG(INFO) << "CURRENT MAIN TIMESTAMP: "
+             << storage_->last_commit_timestamp_.load();
+  if (current_commit_timestamp == storage_->last_commit_timestamp_.load()) {
     DLOG(INFO) << "REPLICA UP TO DATE";
-    replica_state_.store(ReplicaState::READY);
+    std::unique_lock client_guard{client_lock_};
+    replica_state_.store(replication::ReplicaState::READY);
   } else {
     DLOG(INFO) << "REPLICA IS BEHIND";
-    replica_state_.store(ReplicaState::RECOVERY);
+    {
+      std::unique_lock client_guard{client_lock_};
+      replica_state_.store(replication::ReplicaState::RECOVERY);
+    }
     thread_pool_.AddTask(
         [=, this] { this->RecoverReplica(current_commit_timestamp); });
   }
 }
 
-SnapshotRes ReplicationClient::TransferSnapshot(
+SnapshotRes Storage::ReplicationClient::TransferSnapshot(
     const std::filesystem::path &path) {
   auto stream{rpc_client_.Stream<SnapshotRpc>()};
-  Encoder encoder(stream.GetBuilder());
+  replication::Encoder encoder(stream.GetBuilder());
   encoder.WriteFile(path);
   return stream.AwaitResponse();
 }
 
-WalFilesRes ReplicationClient::TransferWalFiles(
+WalFilesRes Storage::ReplicationClient::TransferWalFiles(
     const std::vector<std::filesystem::path> &wal_files) {
   CHECK(!wal_files.empty()) << "Wal files list is empty!";
   auto stream{rpc_client_.Stream<WalFilesRpc>(wal_files.size())};
-  Encoder encoder(stream.GetBuilder());
+  replication::Encoder encoder(stream.GetBuilder());
   for (const auto &wal : wal_files) {
     DLOG(INFO) << "Sending wal file: " << wal;
     encoder.WriteFile(wal);
@@ -79,47 +85,56 @@ WalFilesRes ReplicationClient::TransferWalFiles(
   return stream.AwaitResponse();
 }
 
-OnlySnapshotRes ReplicationClient::TransferOnlySnapshot(
+OnlySnapshotRes Storage::ReplicationClient::TransferOnlySnapshot(
     const uint64_t snapshot_timestamp) {
   auto stream{rpc_client_.Stream<OnlySnapshotRpc>(snapshot_timestamp)};
+  replication::Encoder encoder{stream.GetBuilder()};
+  encoder.WriteString(storage_->epoch_id_);
   return stream.AwaitResponse();
 }
 
-bool ReplicationClient::StartTransactionReplication(
+bool Storage::ReplicationClient::StartTransactionReplication(
     const uint64_t current_wal_seq_num) {
   std::unique_lock guard(client_lock_);
   const auto status = replica_state_.load();
   switch (status) {
-    case ReplicaState::RECOVERY:
+    case replication::ReplicaState::RECOVERY:
       DLOG(INFO) << "Replica " << name_ << " is behind MAIN instance";
       return false;
-    case ReplicaState::REPLICATING:
+    case replication::ReplicaState::REPLICATING:
       DLOG(INFO) << "Replica missed a transaction, going to recovery";
-      replica_state_.store(ReplicaState::RECOVERY);
+      replica_state_.store(replication::ReplicaState::RECOVERY);
       // If it's in replicating state, it should have been up to date with all
       // the commits until now so the replica should contain the
       // last_commit_timestamp
-      thread_pool_.AddTask(
-          [=, this] { this->RecoverReplica(last_commit_timestamp_.load()); });
+      thread_pool_.AddTask([=, this] {
+        this->RecoverReplica(storage_->last_commit_timestamp_.load());
+      });
       return false;
-    case ReplicaState::READY:
+    case replication::ReplicaState::INVALID:
+      LOG(ERROR) << "Couldn't replicate data to " << name_;
+      return false;
+    case replication::ReplicaState::READY:
       CHECK(!replica_stream_);
       try {
-        replica_stream_.emplace(ReplicaStream{
-            this, last_commit_timestamp_.load(), current_wal_seq_num});
+        replica_stream_.emplace(
+            ReplicaStream{this, storage_->last_commit_timestamp_.load(),
+                          current_wal_seq_num});
       } catch (const rpc::RpcFailedException &) {
+        replica_state_.store(replication::ReplicaState::INVALID);
         LOG(ERROR) << "Couldn't replicate data to " << name_;
         thread_pool_.AddTask([this] {
           rpc_client_.Abort();
           InitializeClient();
         });
+        return false;
       }
-      replica_state_.store(ReplicaState::REPLICATING);
+      replica_state_.store(replication::ReplicaState::REPLICATING);
       return true;
   }
 }
 
-void ReplicationClient::IfStreamingTransaction(
+void Storage::ReplicationClient::IfStreamingTransaction(
     const std::function<void(ReplicaStream &handler)> &callback) {
   if (replica_stream_) {
     try {
@@ -134,8 +149,8 @@ void ReplicationClient::IfStreamingTransaction(
   }
 }
 
-void ReplicationClient::FinalizeTransactionReplication() {
-  if (mode_ == ReplicationMode::ASYNC) {
+void Storage::ReplicationClient::FinalizeTransactionReplication() {
+  if (mode_ == replication::ReplicationMode::ASYNC) {
     thread_pool_.AddTask(
         [this] { this->FinalizeTransactionReplicationInternal(); });
   } else {
@@ -143,18 +158,25 @@ void ReplicationClient::FinalizeTransactionReplication() {
   }
 }
 
-void ReplicationClient::FinalizeTransactionReplicationInternal() {
+void Storage::ReplicationClient::FinalizeTransactionReplicationInternal() {
   if (replica_stream_) {
     try {
       auto response = replica_stream_->Finalize();
       if (!response.success) {
-        replica_state_.store(ReplicaState::RECOVERY);
+        {
+          std::unique_lock client_guard{client_lock_};
+          replica_state_.store(replication::ReplicaState::RECOVERY);
+        }
         thread_pool_.AddTask([&, this] {
           this->RecoverReplica(response.current_commit_timestamp);
         });
       }
     } catch (const rpc::RpcFailedException &) {
       LOG(ERROR) << "Couldn't replicate data to " << name_;
+      {
+        std::unique_lock client_guard{client_lock_};
+        replica_state_.store(replication::ReplicaState::INVALID);
+      }
       thread_pool_.AddTask([this] {
         rpc_client_.Abort();
         InitializeClient();
@@ -164,14 +186,14 @@ void ReplicationClient::FinalizeTransactionReplicationInternal() {
   }
 
   std::unique_lock guard(client_lock_);
-  if (replica_state_.load() == ReplicaState::REPLICATING) {
-    replica_state_.store(ReplicaState::READY);
+  if (replica_state_.load() == replication::ReplicaState::REPLICATING) {
+    replica_state_.store(replication::ReplicaState::READY);
   }
 }
 
-void ReplicationClient::RecoverReplica(uint64_t replica_commit) {
+void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
   while (true) {
-    auto file_locker = file_retainer_->AddLocker();
+    auto file_locker = storage_->file_retainer_.AddLocker();
 
     const auto steps = GetRecoverySteps(replica_commit, &file_locker);
     for (const auto &recovery_step : steps) {
@@ -189,17 +211,17 @@ void ReplicationClient::RecoverReplica(uint64_t replica_commit) {
               replica_commit = response.current_commit_timestamp;
               DLOG(INFO) << "CURRENT TIMESTAMP ON REPLICA: " << replica_commit;
             } else if constexpr (std::is_same_v<StepType, RecoveryCurrentWal>) {
-              auto &wal_file = *wal_file_ptr_;
-              std::unique_lock transaction_guard(*transaction_engine_lock_);
-              if (wal_file &&
-                  wal_file->SequenceNumber() == arg.current_wal_seq_num) {
-                wal_file->DisableFlushing();
+              std::unique_lock transaction_guard(storage_->engine_lock_);
+              if (storage_->wal_file_ &&
+                  storage_->wal_file_->SequenceNumber() ==
+                      arg.current_wal_seq_num) {
+                storage_->wal_file_->DisableFlushing();
                 transaction_guard.unlock();
                 DLOG(INFO) << "Sending current wal file";
                 replica_commit = ReplicateCurrentWal();
                 DLOG(INFO) << "CURRENT TIMESTAMP ON REPLICA: "
                            << replica_commit;
-                wal_file->EnableFlushing();
+                storage_->wal_file_->EnableFlushing();
               }
             } else if constexpr (std::is_same_v<StepType,
                                                 RecoveryFinalSnapshot>) {
@@ -216,20 +238,21 @@ void ReplicationClient::RecoverReplica(uint64_t replica_commit) {
           recovery_step);
     }
 
-    if (last_commit_timestamp_.load() == replica_commit) {
-      replica_state_.store(ReplicaState::READY);
+    if (storage_->last_commit_timestamp_.load() == replica_commit) {
+      std::unique_lock client_guard{client_lock_};
+      replica_state_.store(replication::ReplicaState::READY);
       return;
     }
   }
 }
 
-uint64_t ReplicationClient::ReplicateCurrentWal() {
-  auto &wal_file = *wal_file_ptr_;
+uint64_t Storage::ReplicationClient::ReplicateCurrentWal() {
   auto stream = TransferCurrentWalFile();
-  stream.AppendFilename(wal_file->Path().filename());
+  stream.AppendFilename(storage_->wal_file_->Path().filename());
   utils::InputFile file;
-  CHECK(file.Open(wal_file->Path())) << "Failed to open current WAL file!";
-  const auto [buffer, buffer_size] = wal_file->CurrentFileBuffer();
+  CHECK(file.Open(storage_->wal_file_->Path()))
+      << "Failed to open current WAL file!";
+  const auto [buffer, buffer_size] = storage_->wal_file_->CurrentFileBuffer();
   stream.AppendSize(file.GetSize() + buffer_size);
   stream.AppendFileData(&file);
   stream.AppendBufferData(buffer, buffer_size);
@@ -259,26 +282,26 @@ uint64_t ReplicationClient::ReplicateCurrentWal() {
 /// change (creation of that snapshot) the latest timestamp is contained in it.
 /// As no changes were made to the data, we only need to send the timestamp of
 /// the snapshot so replica can set its last timestamp to that value.
-std::vector<ReplicationClient::RecoveryStep>
-ReplicationClient::GetRecoverySteps(
+std::vector<Storage::ReplicationClient::RecoveryStep>
+Storage::ReplicationClient::GetRecoverySteps(
     const uint64_t replica_commit,
     utils::FileRetainer::FileLocker *file_locker) {
-  auto &wal_file = *wal_file_ptr_;
   // First check if we can recover using the current wal file only
   // otherwise save the seq_num of the current wal file
   // This lock is also necessary to force the missed transaction to finish.
   std::optional<uint64_t> current_wal_seq_num;
-  if (std::unique_lock transtacion_guard(*transaction_engine_lock_); wal_file) {
-    current_wal_seq_num.emplace(wal_file->SequenceNumber());
+  if (std::unique_lock transtacion_guard(storage_->engine_lock_);
+      storage_->wal_file_) {
+    current_wal_seq_num.emplace(storage_->wal_file_->SequenceNumber());
   }
 
   auto locker_acc = file_locker->Access();
-  auto wal_files =
-      durability::GetWalFiles(wal_directory_, uuid_, current_wal_seq_num);
+  auto wal_files = durability::GetWalFiles(
+      storage_->wal_directory_, storage_->uuid_, current_wal_seq_num);
   CHECK(wal_files) << "Wal files could not be loaded";
 
-  auto snapshot_files =
-      durability::GetSnapshotFiles(snapshot_directory_, uuid_);
+  auto snapshot_files = durability::GetSnapshotFiles(
+      storage_->snapshot_directory_, storage_->uuid_);
   std::optional<durability::SnapshotDurabilityInfo> latest_snapshot;
   if (!snapshot_files.empty()) {
     std::sort(snapshot_files.begin(), snapshot_files.end());
@@ -404,73 +427,79 @@ ReplicationClient::GetRecoverySteps(
 }
 
 ////// ReplicaStream //////
-ReplicationClient::ReplicaStream::ReplicaStream(
+Storage::ReplicationClient::ReplicaStream::ReplicaStream(
     ReplicationClient *self, const uint64_t previous_commit_timestamp,
     const uint64_t current_seq_num)
     : self_(self),
       stream_(self_->rpc_client_.Stream<AppendDeltasRpc>(
-          previous_commit_timestamp, current_seq_num)) {}
+          previous_commit_timestamp, current_seq_num)) {
+  replication::Encoder encoder{stream_.GetBuilder()};
+  encoder.WriteString(self_->storage_->epoch_id_);
+}
 
-void ReplicationClient::ReplicaStream::AppendDelta(
+void Storage::ReplicationClient::ReplicaStream::AppendDelta(
     const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp) {
-  Encoder encoder(stream_.GetBuilder());
-  EncodeDelta(&encoder, self_->name_id_mapper_, self_->items_, delta, vertex,
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeDelta(&encoder, &self_->storage_->name_id_mapper_,
+              self_->storage_->config_.items, delta, vertex,
               final_commit_timestamp);
 }
 
-void ReplicationClient::ReplicaStream::AppendDelta(
+void Storage::ReplicationClient::ReplicaStream::AppendDelta(
     const Delta &delta, const Edge &edge, uint64_t final_commit_timestamp) {
-  Encoder encoder(stream_.GetBuilder());
-  EncodeDelta(&encoder, self_->name_id_mapper_, delta, edge,
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, delta, edge,
               final_commit_timestamp);
 }
 
-void ReplicationClient::ReplicaStream::AppendTransactionEnd(
+void Storage::ReplicationClient::ReplicaStream::AppendTransactionEnd(
     uint64_t final_commit_timestamp) {
-  Encoder encoder(stream_.GetBuilder());
+  replication::Encoder encoder(stream_.GetBuilder());
   EncodeTransactionEnd(&encoder, final_commit_timestamp);
 }
 
-void ReplicationClient::ReplicaStream::AppendOperation(
+void Storage::ReplicationClient::ReplicaStream::AppendOperation(
     durability::StorageGlobalOperation operation, LabelId label,
     const std::set<PropertyId> &properties, uint64_t timestamp) {
-  Encoder encoder(stream_.GetBuilder());
-  EncodeOperation(&encoder, self_->name_id_mapper_, operation, label,
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeOperation(&encoder, &self_->storage_->name_id_mapper_, operation, label,
                   properties, timestamp);
 }
 
-AppendDeltasRes ReplicationClient::ReplicaStream::Finalize() {
+AppendDeltasRes Storage::ReplicationClient::ReplicaStream::Finalize() {
   return stream_.AwaitResponse();
 }
 
 ////// CurrentWalHandler //////
-ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self)
+Storage::ReplicationClient::CurrentWalHandler::CurrentWalHandler(
+    ReplicationClient *self)
     : self_(self), stream_(self_->rpc_client_.Stream<CurrentWalRpc>()) {}
 
-void ReplicationClient::CurrentWalHandler::AppendFilename(
+void Storage::ReplicationClient::CurrentWalHandler::AppendFilename(
     const std::string &filename) {
-  Encoder encoder(stream_.GetBuilder());
+  replication::Encoder encoder(stream_.GetBuilder());
   encoder.WriteString(filename);
 }
 
-void ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) {
-  Encoder encoder(stream_.GetBuilder());
+void Storage::ReplicationClient::CurrentWalHandler::AppendSize(
+    const size_t size) {
+  replication::Encoder encoder(stream_.GetBuilder());
   encoder.WriteUint(size);
 }
 
-void ReplicationClient::CurrentWalHandler::AppendFileData(
+void Storage::ReplicationClient::CurrentWalHandler::AppendFileData(
     utils::InputFile *file) {
-  Encoder encoder(stream_.GetBuilder());
+  replication::Encoder encoder(stream_.GetBuilder());
   encoder.WriteFileData(file);
 }
 
-void ReplicationClient::CurrentWalHandler::AppendBufferData(
+void Storage::ReplicationClient::CurrentWalHandler::AppendBufferData(
     const uint8_t *buffer, const size_t buffer_size) {
-  Encoder encoder(stream_.GetBuilder());
+  replication::Encoder encoder(stream_.GetBuilder());
   encoder.WriteBuffer(buffer, buffer_size);
 }
 
-CurrentWalRes ReplicationClient::CurrentWalHandler::Finalize() {
+CurrentWalRes Storage::ReplicationClient::CurrentWalHandler::Finalize() {
   return stream_.AwaitResponse();
 }
-}  // namespace storage::replication
+}  // namespace storage

src/storage/v2/replication/replication_client.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <atomic>
-#include <condition_variable>
 #include <thread>
 #include <variant>
 
@@ -13,33 +12,23 @@
 #include "storage/v2/mvcc.hpp"
 #include "storage/v2/name_id_mapper.hpp"
 #include "storage/v2/property_value.hpp"
+#include "storage/v2/replication/enums.hpp"
 #include "storage/v2/replication/rpc.hpp"
 #include "storage/v2/replication/serialization.hpp"
+#include "storage/v2/storage.hpp"
 #include "utils/file.hpp"
 #include "utils/file_locker.hpp"
 #include "utils/spin_lock.hpp"
 #include "utils/synchronized.hpp"
 #include "utils/thread_pool.hpp"
 
-namespace storage::replication {
+namespace storage {
 
-enum class ReplicationMode : std::uint8_t { SYNC, ASYNC };
-
-enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY };
-
-class ReplicationClient {
+class Storage::ReplicationClient {
  public:
-  ReplicationClient(std::string name,
-                    const std::atomic<uint64_t> &last_commit_timestamp,
-                    NameIdMapper *name_id_mapper, Config::Items items,
-                    utils::FileRetainer *file_retainer,
-                    const std::filesystem::path &snapshot_directory,
-                    const std::filesystem::path &wal_directory,
-                    std::string_view uuid,
-                    std::optional<durability::WalFile> *wal_file_ptr,
-                    utils::SpinLock *transaction_engine_lock,
+  ReplicationClient(std::string name, Storage *storage,
                     const io::network::Endpoint &endpoint, bool use_ssl,
-                    ReplicationMode mode);
+                    replication::ReplicationMode mode);
 
   // Handler used for transfering the current transaction.
   class ReplicaStream {
@@ -158,26 +147,19 @@ class ReplicationClient {
   void InitializeClient();
 
   std::string name_;
-  // storage info
-  const std::atomic<uint64_t> &last_commit_timestamp_;
-  NameIdMapper *name_id_mapper_;
-  Config::Items items_;
-  utils::FileRetainer *file_retainer_;
-  const std::filesystem::path &snapshot_directory_;
-  const std::filesystem::path &wal_directory_;
-  std::string_view uuid_;
-  std::optional<durability::WalFile> *wal_file_ptr_;
-  utils::SpinLock *transaction_engine_lock_;
+
+  Storage *storage_;
 
   communication::ClientContext rpc_context_;
   rpc::Client rpc_client_;
 
   std::optional<ReplicaStream> replica_stream_;
-  ReplicationMode mode_{ReplicationMode::SYNC};
+  replication::ReplicationMode mode_{replication::ReplicationMode::SYNC};
 
   utils::SpinLock client_lock_;
   utils::ThreadPool thread_pool_{1};
-  std::atomic<ReplicaState> replica_state_;
+  std::atomic<replication::ReplicaState> replica_state_{
+      replication::ReplicaState::INVALID};
 };
 
-}  // namespace storage::replication
+}  // namespace storage

src/storage/v2/replication/replication_server.cpp

@@ -0,0 +1,727 @@
+#include "storage/v2/replication/replication_server.hpp"
+
+#include "storage/v2/durability/durability.hpp"
+#include "storage/v2/durability/snapshot.hpp"
+#include "storage/v2/transaction.hpp"
+#include "utils/exceptions.hpp"
+
+namespace storage {
+Storage::ReplicationServer::ReplicationServer(Storage *storage,
+                                              io::network::Endpoint endpoint)
+    : storage_(storage) {
+  // Create RPC server.
+  // TODO (antonio2368): Add support for SSL.
+  rpc_server_context_.emplace();
+  // NOTE: The replication server must have a single thread for processing
+  // because there is no need for more processing threads - each replica can
+  // have only a single main server. Also, the single-threaded guarantee
+  // simplifies the rest of the implementation.
+  rpc_server_.emplace(std::move(endpoint), &*rpc_server_context_,
+                      /* workers_count = */ 1);
+
+  rpc_server_->Register<HeartbeatRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received HeartbeatRpc";
+        this->HeartbeatHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<AppendDeltasRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received AppendDeltasRpc:";
+        this->AppendDeltasHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<SnapshotRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received SnapshotRpc";
+        this->SnapshotHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<OnlySnapshotRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received OnlySnapshotRpc";
+        this->OnlySnapshotHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<WalFilesRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received WalFilesRpc";
+        this->WalFilesHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<CurrentWalRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received CurrentWalRpc";
+        this->CurrentWalHandler(req_reader, res_builder);
+      });
+  rpc_server_->Start();
+}
+
+void Storage::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader,
+                                                  slk::Builder *res_builder) {
+  HeartbeatReq req;
+  slk::Load(&req, req_reader);
+  replication::Decoder decoder{req_reader};
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid value read form HeartbeatRpc!";
+  if (storage_->last_commit_timestamp_ == kTimestampInitialId) {
+    // The replica has no commits
+    // use the main's epoch id
+    storage_->epoch_id_ = std::move(*maybe_epoch_id);
+  } else if (*maybe_epoch_id != storage_->epoch_id_) {
+    auto &epoch_history = storage_->epoch_history_;
+    const auto result =
+        std::find_if(epoch_history.rbegin(), epoch_history.rend(),
+                     [&](const auto &epoch_info) {
+                       return epoch_info.first == *maybe_epoch_id;
+                     });
+    auto branching_point = kTimestampInitialId;
+    if (result == epoch_history.rend()) {
+      // we couldn't find the epoch_id inside the history so if it has
+      // the same or larger commit timestamp, some old replica became a main
+      // This isn't always the case, there is one case where an old main
+      // becomes a replica then main again and it should have a commit timestamp
+      // larger than the one on replica.
+      if (req.main_commit_timestamp >= storage_->last_commit_timestamp_) {
+        epoch_history.emplace_back(std::move(storage_->epoch_id_),
+                                   storage_->last_commit_timestamp_);
+        storage_->epoch_id_ = std::move(*maybe_epoch_id);
+        HeartbeatRes res{true, storage_->last_commit_timestamp_.load()};
+        slk::Save(res, res_builder);
+        return;
+      }
+    } else {
+      branching_point = result->second;
+    }
+    HeartbeatRes res{false, branching_point};
+    slk::Save(res, res_builder);
+    return;
+  }
+  HeartbeatRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::AppendDeltasHandler(
+    slk::Reader *req_reader, slk::Builder *res_builder) {
+  AppendDeltasReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid replication message";
+
+  const auto read_delta =
+      [&]() -> std::pair<uint64_t, durability::WalDeltaData> {
+    try {
+      auto timestamp = ReadWalDeltaHeader(&decoder);
+      DLOG(INFO) << "       Timestamp " << timestamp;
+      auto delta = ReadWalDeltaData(&decoder);
+      return {timestamp, delta};
+    } catch (const slk::SlkReaderException &) {
+      throw utils::BasicException("Missing data!");
+    } catch (const durability::RecoveryFailure &) {
+      throw utils::BasicException("Invalid data!");
+    }
+  };
+
+  // TODO (antonio2368): Add error handling for different epoch id
+  if (*maybe_epoch_id != storage_->epoch_id_) {
+    throw utils::BasicException("Invalid epoch id");
+  }
+
+  if (req.previous_commit_timestamp !=
+      storage_->last_commit_timestamp_.load()) {
+    // Empty the stream
+    bool transaction_complete = false;
+    while (!transaction_complete) {
+      DLOG(INFO) << "Skipping delta";
+      const auto [timestamp, delta] = read_delta();
+      transaction_complete =
+          durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
+    }
+
+    AppendDeltasRes res{false, storage_->last_commit_timestamp_.load()};
+    slk::Save(res, res_builder);
+    return;
+  }
+
+  if (storage_->wal_file_) {
+    if (req.seq_num > storage_->wal_file_->SequenceNumber() ||
+        *maybe_epoch_id != storage_->epoch_id_) {
+      storage_->wal_file_->FinalizeWal();
+      storage_->wal_file_.reset();
+      storage_->wal_seq_num_ = req.seq_num;
+    } else {
+      CHECK(storage_->wal_file_->SequenceNumber() == req.seq_num)
+          << "Invalid sequence number of current wal file";
+      storage_->wal_seq_num_ = req.seq_num + 1;
+    }
+  } else {
+    storage_->wal_seq_num_ = req.seq_num;
+  }
+
+  auto edge_acc = storage_->edges_.access();
+  auto vertex_acc = storage_->vertices_.access();
+
+  std::optional<std::pair<uint64_t, storage::Storage::Accessor>>
+      commit_timestamp_and_accessor;
+  auto get_transaction =
+      [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
+        if (!commit_timestamp_and_accessor) {
+          commit_timestamp_and_accessor.emplace(commit_timestamp,
+                                                storage_->Access());
+        } else if (commit_timestamp_and_accessor->first != commit_timestamp) {
+          throw utils::BasicException("Received more than one transaction!");
+        }
+        return &commit_timestamp_and_accessor->second;
+      };
+
+  bool transaction_complete = false;
+  for (uint64_t i = 0; !transaction_complete; ++i) {
+    DLOG(INFO) << "  Delta " << i;
+    const auto [timestamp, delta] = read_delta();
+
+    switch (delta.type) {
+      case durability::WalDeltaData::Type::VERTEX_CREATE: {
+        DLOG(INFO) << "       Create vertex "
+                   << delta.vertex_create_delete.gid.AsUint();
+        auto transaction = get_transaction(timestamp);
+        transaction->CreateVertex(delta.vertex_create_delete.gid);
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_DELETE: {
+        DLOG(INFO) << "       Delete vertex "
+                   << delta.vertex_create_delete.gid.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = transaction->DeleteVertex(&*vertex);
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_add_remove_label.gid.AsUint()
+                   << " add label " << delta.vertex_add_remove_label.label;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = vertex->AddLabel(
+            transaction->NameToLabel(delta.vertex_add_remove_label.label));
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_add_remove_label.gid.AsUint()
+                   << " remove label " << delta.vertex_add_remove_label.label;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = vertex->RemoveLabel(
+            transaction->NameToLabel(delta.vertex_add_remove_label.label));
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_edge_set_property.gid.AsUint()
+                   << " set property "
+                   << delta.vertex_edge_set_property.property << " to "
+                   << delta.vertex_edge_set_property.value;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(
+            delta.vertex_edge_set_property.gid, storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret =
+            vertex->SetProperty(transaction->NameToProperty(
+                                    delta.vertex_edge_set_property.property),
+                                delta.vertex_edge_set_property.value);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_CREATE: {
+        DLOG(INFO) << "       Create edge "
+                   << delta.edge_create_delete.gid.AsUint() << " of type "
+                   << delta.edge_create_delete.edge_type << " from vertex "
+                   << delta.edge_create_delete.from_vertex.AsUint()
+                   << " to vertex "
+                   << delta.edge_create_delete.to_vertex.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto from_vertex = transaction->FindVertex(
+            delta.edge_create_delete.from_vertex, storage::View::NEW);
+        if (!from_vertex) throw utils::BasicException("Invalid transaction!");
+        auto to_vertex = transaction->FindVertex(
+            delta.edge_create_delete.to_vertex, storage::View::NEW);
+        if (!to_vertex) throw utils::BasicException("Invalid transaction!");
+        auto edge = transaction->CreateEdge(
+            &*from_vertex, &*to_vertex,
+            transaction->NameToEdgeType(delta.edge_create_delete.edge_type),
+            delta.edge_create_delete.gid);
+        if (edge.HasError())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_DELETE: {
+        DLOG(INFO) << "       Delete edge "
+                   << delta.edge_create_delete.gid.AsUint() << " of type "
+                   << delta.edge_create_delete.edge_type << " from vertex "
+                   << delta.edge_create_delete.from_vertex.AsUint()
+                   << " to vertex "
+                   << delta.edge_create_delete.to_vertex.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto from_vertex = transaction->FindVertex(
+            delta.edge_create_delete.from_vertex, storage::View::NEW);
+        if (!from_vertex) throw utils::BasicException("Invalid transaction!");
+        auto to_vertex = transaction->FindVertex(
+            delta.edge_create_delete.to_vertex, storage::View::NEW);
+        if (!to_vertex) throw utils::BasicException("Invalid transaction!");
+        auto edges = from_vertex->OutEdges(
+            storage::View::NEW,
+            {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)},
+            &*to_vertex);
+        if (edges.HasError())
+          throw utils::BasicException("Invalid transaction!");
+        if (edges->size() != 1)
+          throw utils::BasicException("Invalid transaction!");
+        auto &edge = (*edges)[0];
+        auto ret = transaction->DeleteEdge(&edge);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
+        DLOG(INFO) << "       Edge "
+                   << delta.vertex_edge_set_property.gid.AsUint()
+                   << " set property "
+                   << delta.vertex_edge_set_property.property << " to "
+                   << delta.vertex_edge_set_property.value;
+
+        if (!storage_->config_.items.properties_on_edges)
+          throw utils::BasicException(
+              "Can't set properties on edges because properties on edges "
+              "are disabled!");
+
+        auto transaction = get_transaction(timestamp);
+
+        // The following block of code effectively implements `FindEdge` and
+        // yields an accessor that is only valid for managing the edge's
+        // properties.
+        auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
+        if (edge == edge_acc.end())
+          throw utils::BasicException("Invalid transaction!");
+        // The edge visibility check must be done here manually because we
+        // don't allow direct access to the edges through the public API.
+        {
+          bool is_visible = true;
+          Delta *delta = nullptr;
+          {
+            std::lock_guard<utils::SpinLock> guard(edge->lock);
+            is_visible = !edge->deleted;
+            delta = edge->delta;
+          }
+          ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW,
+                             [&is_visible](const Delta &delta) {
+                               switch (delta.action) {
+                                 case Delta::Action::ADD_LABEL:
+                                 case Delta::Action::REMOVE_LABEL:
+                                 case Delta::Action::SET_PROPERTY:
+                                 case Delta::Action::ADD_IN_EDGE:
+                                 case Delta::Action::ADD_OUT_EDGE:
+                                 case Delta::Action::REMOVE_IN_EDGE:
+                                 case Delta::Action::REMOVE_OUT_EDGE:
+                                   break;
+                                 case Delta::Action::RECREATE_OBJECT: {
+                                   is_visible = true;
+                                   break;
+                                 }
+                                 case Delta::Action::DELETE_OBJECT: {
+                                   is_visible = false;
+                                   break;
+                                 }
+                               }
+                             });
+          if (!is_visible) throw utils::BasicException("Invalid transaction!");
+        }
+        EdgeRef edge_ref(&*edge);
+        // Here we create an edge accessor that we will use to get the
+        // properties of the edge. The accessor is created with an invalid
+        // type and invalid from/to pointers because we don't know them
+        // here, but that isn't an issue because we won't use that part of
+        // the API here.
+        auto ea = EdgeAccessor{edge_ref,
+                               EdgeTypeId::FromUint(0UL),
+                               nullptr,
+                               nullptr,
+                               &transaction->transaction_,
+                               &storage_->indices_,
+                               &storage_->constraints_,
+                               storage_->config_.items};
+
+        auto ret = ea.SetProperty(transaction->NameToProperty(
+                                      delta.vertex_edge_set_property.property),
+                                  delta.vertex_edge_set_property.value);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+
+      case durability::WalDeltaData::Type::TRANSACTION_END: {
+        DLOG(INFO) << "       Transaction end";
+        if (!commit_timestamp_and_accessor ||
+            commit_timestamp_and_accessor->first != timestamp)
+          throw utils::BasicException("Invalid data!");
+        auto ret = commit_timestamp_and_accessor->second.Commit(
+            commit_timestamp_and_accessor->first);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        commit_timestamp_and_accessor = std::nullopt;
+        break;
+      }
+
+      case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
+        DLOG(INFO) << "       Create label index on :"
+                   << delta.operation_label.label;
+        // Need to send the timestamp
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->CreateIndex(
+                storage_->NameToLabel(delta.operation_label.label), timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
+        DLOG(INFO) << "       Drop label index on :"
+                   << delta.operation_label.label;
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropIndex(
+                storage_->NameToLabel(delta.operation_label.label), timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
+        DLOG(INFO) << "       Create label+property index on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->CreateIndex(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
+        DLOG(INFO) << "       Drop label+property index on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropIndex(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
+        DLOG(INFO) << "       Create existence constraint on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        auto ret = storage_->CreateExistenceConstraint(
+            storage_->NameToLabel(delta.operation_label_property.label),
+            storage_->NameToProperty(delta.operation_label_property.property),
+            timestamp);
+        if (!ret.HasValue() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
+        DLOG(INFO) << "       Drop existence constraint on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropExistenceConstraint(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
+        std::stringstream ss;
+        utils::PrintIterable(ss, delta.operation_label_properties.properties);
+        DLOG(INFO) << "       Create unique constraint on :"
+                   << delta.operation_label_properties.label << " (" << ss.str()
+                   << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        std::set<PropertyId> properties;
+        for (const auto &prop : delta.operation_label_properties.properties) {
+          properties.emplace(storage_->NameToProperty(prop));
+        }
+        auto ret = storage_->CreateUniqueConstraint(
+            storage_->NameToLabel(delta.operation_label_properties.label),
+            properties, timestamp);
+        if (!ret.HasValue() ||
+            ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
+        std::stringstream ss;
+        utils::PrintIterable(ss, delta.operation_label_properties.properties);
+        DLOG(INFO) << "       Drop unique constraint on :"
+                   << delta.operation_label_properties.label << " (" << ss.str()
+                   << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        std::set<PropertyId> properties;
+        for (const auto &prop : delta.operation_label_properties.properties) {
+          properties.emplace(storage_->NameToProperty(prop));
+        }
+        auto ret = storage_->DropUniqueConstraint(
+            storage_->NameToLabel(delta.operation_label_properties.label),
+            properties, timestamp);
+        if (ret != UniqueConstraints::DeletionStatus::SUCCESS)
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+    }
+    transaction_complete =
+        durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
+  }
+
+  if (commit_timestamp_and_accessor)
+    throw utils::BasicException("Invalid data!");
+
+  AppendDeltasRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader,
+                                                 slk::Builder *res_builder) {
+  SnapshotReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->snapshot_directory_);
+
+  const auto maybe_snapshot_path =
+      decoder.ReadFile(storage_->snapshot_directory_);
+  CHECK(maybe_snapshot_path) << "Failed to load snapshot!";
+  DLOG(INFO) << "Received snapshot saved to " << *maybe_snapshot_path;
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  // Clear the database
+  storage_->vertices_.clear();
+  storage_->edges_.clear();
+
+  storage_->constraints_ = Constraints();
+  // TODO (antonio2368): Check if there's a less hacky way
+  storage_->indices_.label_index = LabelIndex(
+      &storage_->indices_, &storage_->constraints_, storage_->config_.items);
+  storage_->indices_.label_property_index = LabelPropertyIndex(
+      &storage_->indices_, &storage_->constraints_, storage_->config_.items);
+  try {
+    DLOG(INFO) << "Loading snapshot";
+    auto recovered_snapshot = durability::LoadSnapshot(
+        *maybe_snapshot_path, &storage_->vertices_, &storage_->edges_,
+        &storage_->epoch_history_, &storage_->name_id_mapper_,
+        &storage_->edge_count_, storage_->config_.items);
+    DLOG(INFO) << "Snapshot loaded successfully";
+    // If this step is present it should always be the first step of
+    // the recovery so we use the UUID we read from snasphost
+    storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
+    storage_->epoch_id_ = std::move(recovered_snapshot.snapshot_info.epoch_id);
+    const auto &recovery_info = recovered_snapshot.recovery_info;
+    storage_->vertex_id_ = recovery_info.next_vertex_id;
+    storage_->edge_id_ = recovery_info.next_edge_id;
+    storage_->timestamp_ =
+        std::max(storage_->timestamp_, recovery_info.next_timestamp);
+
+    durability::RecoverIndicesAndConstraints(
+        recovered_snapshot.indices_constraints, &storage_->indices_,
+        &storage_->constraints_, &storage_->vertices_);
+  } catch (const durability::RecoveryFailure &e) {
+    // TODO (antonio2368): What to do if the sent snapshot is invalid
+    LOG(WARNING) << "Couldn't load the snapshot because of: " << e.what();
+  }
+  storage_->last_commit_timestamp_ = storage_->timestamp_ - 1;
+  storage_guard.unlock();
+
+  SnapshotRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+
+  // Delete other durability files
+  auto snapshot_files = durability::GetSnapshotFiles(
+      storage_->snapshot_directory_, storage_->uuid_);
+  for (const auto &[path, uuid, _] : snapshot_files) {
+    if (path != *maybe_snapshot_path) {
+      storage_->file_retainer_.DeleteFile(path);
+    }
+  }
+
+  auto wal_files =
+      durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_);
+  if (wal_files) {
+    for (const auto &wal_file : *wal_files) {
+      storage_->file_retainer_.DeleteFile(wal_file.path);
+    }
+
+    storage_->wal_file_.reset();
+  }
+}
+
+void Storage::ReplicationServer::OnlySnapshotHandler(
+    slk::Reader *req_reader, slk::Builder *res_builder) {
+  OnlySnapshotReq req;
+  slk::Load(&req, req_reader);
+
+  CHECK(storage_->last_commit_timestamp_.load() < req.snapshot_timestamp)
+      << "Invalid snapshot timestamp, it should be less than the last"
+         "commited timestamp";
+
+  replication::Decoder decoder{req_reader};
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid replication message";
+
+  if (*maybe_epoch_id != storage_->epoch_id_) {
+    storage_->epoch_history_.emplace_back(std::move(storage_->epoch_id_),
+                                          storage_->last_commit_timestamp_);
+    storage_->epoch_id_ = std::move(*maybe_epoch_id);
+  }
+
+  storage_->last_commit_timestamp_.store(req.snapshot_timestamp);
+
+  OnlySnapshotRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader,
+                                                 slk::Builder *res_builder) {
+  WalFilesReq req;
+  slk::Load(&req, req_reader);
+
+  const auto wal_file_number = req.file_number;
+  DLOG(INFO) << "Received WAL files: " << wal_file_number;
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->wal_directory_);
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  durability::RecoveredIndicesAndConstraints indices_constraints;
+  auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
+  if (wal_info.seq_num == 0) {
+    storage_->uuid_ = wal_info.uuid;
+  }
+
+  // Check the seq number of the first wal file to see if it's the
+  // finalized form of the current wal on replica
+  if (storage_->wal_file_) {
+    if (storage_->wal_file_->SequenceNumber() == wal_info.seq_num &&
+        storage_->wal_file_->Path() != path) {
+      storage_->wal_file_->DeleteWal();
+    }
+    storage_->wal_file_.reset();
+  }
+
+  for (auto i = 1; i < wal_file_number; ++i) {
+    LoadWal(&decoder, &indices_constraints);
+  }
+
+  durability::RecoverIndicesAndConstraints(
+      indices_constraints, &storage_->indices_, &storage_->constraints_,
+      &storage_->vertices_);
+  storage_guard.unlock();
+
+  WalFilesRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader,
+                                                   slk::Builder *res_builder) {
+  CurrentWalReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->wal_directory_);
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  durability::RecoveredIndicesAndConstraints indices_constraints;
+  auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
+  if (wal_info.seq_num == 0) {
+    storage_->uuid_ = wal_info.uuid;
+  }
+
+  if (storage_->wal_file_ &&
+      storage_->wal_file_->SequenceNumber() == wal_info.seq_num &&
+      storage_->wal_file_->Path() != path) {
+    // Delete the old wal file
+    storage_->file_retainer_.DeleteFile(storage_->wal_file_->Path());
+  }
+  CHECK(storage_->config_.durability.snapshot_wal_mode ==
+        Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL);
+  storage_->wal_file_.emplace(std::move(path), storage_->config_.items,
+                              &storage_->name_id_mapper_, wal_info.seq_num,
+                              wal_info.from_timestamp, wal_info.to_timestamp,
+                              wal_info.num_deltas, &storage_->file_retainer_);
+  durability::RecoverIndicesAndConstraints(
+      indices_constraints, &storage_->indices_, &storage_->constraints_,
+      &storage_->vertices_);
+  storage_guard.unlock();
+
+  CurrentWalRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+std::pair<durability::WalInfo, std::filesystem::path>
+Storage::ReplicationServer::LoadWal(
+    replication::Decoder *decoder,
+    durability::RecoveredIndicesAndConstraints *indices_constraints) {
+  auto maybe_wal_path = decoder->ReadFile(storage_->wal_directory_, "_MAIN");
+  CHECK(maybe_wal_path) << "Failed to load WAL!";
+  DLOG(INFO) << "Received WAL saved to " << *maybe_wal_path;
+  try {
+    auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
+    if (wal_info.epoch_id != storage_->epoch_id_) {
+      storage_->epoch_history_.emplace_back(wal_info.epoch_id,
+                                            storage_->last_commit_timestamp_);
+      storage_->epoch_id_ = std::move(wal_info.epoch_id);
+    }
+    auto info = durability::LoadWal(
+        *maybe_wal_path, indices_constraints, storage_->last_commit_timestamp_,
+        &storage_->vertices_, &storage_->edges_, &storage_->name_id_mapper_,
+        &storage_->edge_count_, storage_->config_.items);
+    storage_->vertex_id_ =
+        std::max(storage_->vertex_id_.load(), info.next_vertex_id);
+    storage_->edge_id_ = std::max(storage_->edge_id_.load(), info.next_edge_id);
+    storage_->timestamp_ = std::max(storage_->timestamp_, info.next_timestamp);
+    if (info.next_timestamp != 0) {
+      storage_->last_commit_timestamp_ = info.next_timestamp - 1;
+    }
+    DLOG(INFO) << *maybe_wal_path << " loaded successfully";
+    return {std::move(wal_info), std::move(*maybe_wal_path)};
+  } catch (const durability::RecoveryFailure &e) {
+    LOG(FATAL) << "Couldn't recover WAL deltas from " << *maybe_wal_path
+               << " because of: " << e.what();
+  }
+}
+
+Storage::ReplicationServer::~ReplicationServer() {
+  if (rpc_server_) {
+    rpc_server_->Shutdown();
+    rpc_server_->AwaitShutdown();
+  }
+}
+}  // namespace storage

src/storage/v2/replication/replication_server.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "storage/v2/storage.hpp"
+
+namespace storage {
+
+class Storage::ReplicationServer {
+ public:
+  explicit ReplicationServer(Storage *storage, io::network::Endpoint endpoint);
+  ReplicationServer(const ReplicationServer &) = delete;
+  ReplicationServer(ReplicationServer &&) = delete;
+  ReplicationServer &operator=(const ReplicationServer &) = delete;
+  ReplicationServer &operator=(ReplicationServer &&) = delete;
+
+  ~ReplicationServer();
+
+ private:
+  // RPC handlers
+  void HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  // RPC for replicating only the commit of the last snapshot as that is the
+  // only difference between the replica and main (all of the data is
+  // already replicated through previous WAL\Snapshot files)
+  void OnlySnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+
+  std::pair<durability::WalInfo, std::filesystem::path> LoadWal(
+      replication::Decoder *decoder,
+      durability::RecoveredIndicesAndConstraints *indices_constraints);
+
+  std::optional<communication::ServerContext> rpc_server_context_;
+  std::optional<rpc::Server> rpc_server_;
+
+  Storage *storage_;
+};
+
+}  // namespace storage

src/storage/v2/replication/rpc.lcp

@@ -23,9 +23,10 @@ cpp<#
      (current-commit-timestamp :uint64_t))))
 
 (lcp:define-rpc heartbeat
-  (:request ())
+  (:request ((main-commit-timestamp :uint64_t)))
   (:response
-    ((current-commit-timestamp :uint64_t))))
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
 
 (lcp:define-rpc snapshot
   (:request ())

src/storage/v2/storage.cpp

@@ -24,6 +24,8 @@
 #include "utils/uuid.hpp"
 
 #ifdef MG_ENTERPRISE
+#include "storage/v2/replication/replication_client.hpp"
+#include "storage/v2/replication/replication_server.hpp"
 #include "storage/v2/replication/rpc.hpp"
 #endif
 
@@ -35,6 +37,10 @@ DEFINE_bool(async_replica, false, "Set to true to be the replica");
 
 namespace storage {
 
+namespace {
+constexpr uint16_t kEpochHistoryRetention = 1000;
+}  // namespace
+
 auto AdvanceToVisibleVertex(utils::SkipList<Vertex>::Iterator it,
                             utils::SkipList<Vertex>::Iterator end,
                             std::optional<VertexAccessor> *vertex,
@@ -323,7 +329,8 @@ Storage::Storage(Config config)
                      durability::kWalDirectory),
       lock_file_path_(config_.durability.storage_directory /
                       durability::kLockFile),
-      uuid_(utils::GenerateUUID()) {
+      uuid_(utils::GenerateUUID()),
+      epoch_id_(utils::GenerateUUID()) {
   if (config_.durability.snapshot_wal_mode !=
           Config::Durability::SnapshotWalMode::DISABLED ||
       config_.durability.snapshot_on_exit ||
@@ -355,9 +362,9 @@ Storage::Storage(Config config)
   }
   if (config_.durability.recover_on_startup) {
     auto info = durability::RecoverData(
-        snapshot_directory_, wal_directory_, &uuid_, &vertices_, &edges_,
-        &edge_count_, &name_id_mapper_, &indices_, &constraints_, config_.items,
-        &wal_seq_num_);
+        snapshot_directory_, wal_directory_, &uuid_, &epoch_id_,
+        &epoch_history_, &vertices_, &edges_, &edge_count_, &name_id_mapper_,
+        &indices_, &constraints_, config_.items, &wal_seq_num_);
     if (info) {
       vertex_id_ = info->next_vertex_id;
       edge_id_ = info->next_edge_id;
@@ -418,7 +425,6 @@ Storage::Storage(Config config)
   // For testing purposes until we can define the instance type from
   // a query.
   if (FLAGS_main) {
-    SetReplicationRole<ReplicationRole::MAIN>();
     RegisterReplica("REPLICA_SYNC", io::network::Endpoint{"127.0.0.1", 10000});
     RegisterReplica("REPLICA_ASYNC", io::network::Endpoint{"127.0.0.1", 10002});
   } else if (FLAGS_replica) {
@@ -1661,8 +1667,8 @@ bool Storage::InitializeWalFile() {
       Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL)
     return false;
   if (!wal_file_) {
-    wal_file_.emplace(wal_directory_, uuid_, config_.items, &name_id_mapper_,
-                      wal_seq_num_++, &file_retainer_);
+    wal_file_.emplace(wal_directory_, uuid_, epoch_id_, config_.items,
+                      &name_id_mapper_, wal_seq_num_++, &file_retainer_);
   }
   return true;
 }
@@ -1697,13 +1703,10 @@ void Storage::AppendToWal(const Transaction &transaction,
       transaction.commit_timestamp->load(std::memory_order_acquire);
 
 #ifdef MG_ENTERPRISE
-  // We need to keep this lock because handler takes a pointer to the client
-  // from which it was created
-  std::shared_lock<utils::RWLock> replication_guard(replication_lock_);
   if (replication_role_.load() == ReplicationRole::MAIN) {
     replication_clients_.WithLock([&](auto &clients) {
       for (auto &client : clients) {
-        client.StartTransactionReplication(wal_file_->SequenceNumber());
+        client->StartTransactionReplication(wal_file_->SequenceNumber());
       }
     });
   }
@@ -1727,7 +1730,7 @@ void Storage::AppendToWal(const Transaction &transaction,
 #ifdef MG_ENTERPRISE
         replication_clients_.WithLock([&](auto &clients) {
           for (auto &client : clients) {
-            client.IfStreamingTransaction([&](auto &stream) {
+            client->IfStreamingTransaction([&](auto &stream) {
               stream.AppendDelta(*delta, parent, final_commit_timestamp);
             });
           }
@@ -1869,10 +1872,10 @@ void Storage::AppendToWal(const Transaction &transaction,
 #ifdef MG_ENTERPRISE
   replication_clients_.WithLock([&](auto &clients) {
     for (auto &client : clients) {
-      client.IfStreamingTransaction([&](auto &stream) {
+      client->IfStreamingTransaction([&](auto &stream) {
         stream.AppendTransactionEnd(final_commit_timestamp);
       });
-      client.FinalizeTransactionReplication();
+      client->FinalizeTransactionReplication();
     }
   });
 #endif
@@ -1886,16 +1889,15 @@ void Storage::AppendToWal(durability::StorageGlobalOperation operation,
                              final_commit_timestamp);
 #ifdef MG_ENTERPRISE
   {
-    std::shared_lock<utils::RWLock> replication_guard(replication_lock_);
     if (replication_role_.load() == ReplicationRole::MAIN) {
       replication_clients_.WithLock([&](auto &clients) {
         for (auto &client : clients) {
-          client.StartTransactionReplication(wal_file_->SequenceNumber());
-          client.IfStreamingTransaction([&](auto &stream) {
+          client->StartTransactionReplication(wal_file_->SequenceNumber());
+          client->IfStreamingTransaction([&](auto &stream) {
             stream.AppendOperation(operation, label, properties,
                                    final_commit_timestamp);
           });
-          client.FinalizeTransactionReplication();
+          client->FinalizeTransactionReplication();
         }
       });
     }
@@ -1922,8 +1924,8 @@ void Storage::CreateSnapshot() {
   durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_,
                              config_.durability.snapshot_retention_count,
                              &vertices_, &edges_, &name_id_mapper_, &indices_,
-                             &constraints_, config_.items, uuid_,
-                             &file_retainer_);
+                             &constraints_, config_.items, uuid_, epoch_id_,
+                             epoch_history_, &file_retainer_);
 
   // Finalize snapshot transaction.
   commit_log_.MarkFinished(transaction.start_timestamp);
@@ -1945,678 +1947,71 @@ uint64_t Storage::CommitTimestamp(
 }
 
 #ifdef MG_ENTERPRISE
-std::pair<durability::WalInfo, std::filesystem::path> Storage::LoadWal(
-    replication::Decoder *decoder,
-    durability::RecoveredIndicesAndConstraints *indices_constraints) {
-  auto maybe_wal_path = decoder->ReadFile(wal_directory_, "_MAIN");
-  CHECK(maybe_wal_path) << "Failed to load WAL!";
-  DLOG(INFO) << "Received WAL saved to " << *maybe_wal_path;
-  try {
-    auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
-    auto info = durability::LoadWal(
-        *maybe_wal_path, indices_constraints, last_commit_timestamp_,
-        &vertices_, &edges_, &name_id_mapper_, &edge_count_, config_.items);
-    vertex_id_ = std::max(vertex_id_.load(), info.next_vertex_id);
-    edge_id_ = std::max(edge_id_.load(), info.next_edge_id);
-    timestamp_ = std::max(timestamp_, info.next_timestamp);
-    if (info.next_timestamp != 0) {
-      last_commit_timestamp_ = info.next_timestamp - 1;
-    }
-    DLOG(INFO) << *maybe_wal_path << " loaded successfully";
-    return {std::move(wal_info), std::move(*maybe_wal_path)};
-  } catch (const durability::RecoveryFailure &e) {
-    LOG(FATAL) << "Couldn't recover WAL deltas from " << *maybe_wal_path
-               << " because of: " << e.what();
-  }
+void Storage::ConfigureReplica(io::network::Endpoint endpoint) {
+  replication_server_ =
+      std::make_unique<ReplicationServer>(this, std::move(endpoint));
 }
 
-void Storage::ConfigureReplica(io::network::Endpoint endpoint) {
-  replication_server_.emplace();
+void Storage::ConfigureMain() {
+  // Main instance does not need replication server
+  // This should be always called first so we finalize everything
+  replication_server_.reset(nullptr);
 
-  // Create RPC server.
-  // TODO (antonio2368): Add support for SSL.
-  replication_server_->rpc_server_context.emplace();
-  // NOTE: The replication server must have a single thread for processing
-  // because there is no need for more processing threads - each replica can
-  // have only a single main server. Also, the single-threaded guarantee
-  // simplifies the rest of the implementation.
-  replication_server_->rpc_server.emplace(
-      endpoint, &*replication_server_->rpc_server_context,
-      /* workers_count = */ 1);
+  std::unique_lock engine_guard{engine_lock_};
+  if (wal_file_) {
+    wal_file_->FinalizeWal();
+    wal_file_.reset();
+  }
 
-  replication_server_->rpc_server->Register<HeartbeatRpc>(
-      [this](auto *req_reader, auto *res_builder) {
-        HeartbeatReq req;
-        slk::Load(&req, req_reader);
-
-        DLOG(INFO) << "Received HeartbeatRpc:";
-
-        HeartbeatRes res{last_commit_timestamp_.load()};
-        slk::Save(res, res_builder);
-      });
-  replication_server_->rpc_server->Register<
-      AppendDeltasRpc>([this](auto *req_reader, auto *res_builder) {
-    AppendDeltasReq req;
-    slk::Load(&req, req_reader);
-
-    DLOG(INFO) << "Received AppendDeltasRpc:";
-
-    constexpr auto is_transaction_complete =
-        [](const durability::WalDeltaData::Type delta_type) {
-          switch (delta_type) {
-            case durability::WalDeltaData::Type::TRANSACTION_END:
-            case durability::WalDeltaData::Type::LABEL_INDEX_CREATE:
-            case durability::WalDeltaData::Type::LABEL_INDEX_DROP:
-            case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
-            case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
-            case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
-            case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
-            case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
-            case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
-              return true;
-            default:
-              return false;
-          }
-        };
-
-    replication::Decoder decoder(req_reader);
-    const auto read_delta =
-        [&]() -> std::pair<uint64_t, durability::WalDeltaData> {
-      try {
-        auto timestamp = ReadWalDeltaHeader(&decoder);
-        DLOG(INFO) << "       Timestamp " << timestamp;
-        auto delta = ReadWalDeltaData(&decoder);
-        return {timestamp, delta};
-      } catch (const slk::SlkReaderException &) {
-        throw utils::BasicException("Missing data!");
-      } catch (const durability::RecoveryFailure &) {
-        throw utils::BasicException("Invalid data!");
-      }
-    };
-
-    if (req.previous_commit_timestamp != last_commit_timestamp_.load()) {
-      // Empty the stream
-      bool transaction_complete = false;
-      while (!transaction_complete) {
-        DLOG(INFO) << "Skipping delta";
-        const auto [timestamp, delta] = read_delta();
-        transaction_complete = is_transaction_complete(delta.type);
-      }
-
-      AppendDeltasRes res{false, last_commit_timestamp_.load()};
-      slk::Save(res, res_builder);
-      return;
-    }
-
-    if (wal_file_) {
-      if (req.seq_num > wal_file_->SequenceNumber()) {
-        wal_file_->FinalizeWal();
-        wal_file_.reset();
-        wal_seq_num_ = req.seq_num;
-      } else {
-        CHECK(wal_file_->SequenceNumber() == req.seq_num)
-            << "Invalid sequence number of current wal file";
-        wal_seq_num_ = req.seq_num + 1;
-      }
-    } else {
-      wal_seq_num_ = req.seq_num;
-    }
-
-    auto edge_acc = edges_.access();
-    auto vertex_acc = vertices_.access();
-
-    std::optional<std::pair<uint64_t, storage::Storage::Accessor>>
-        commit_timestamp_and_accessor;
-    auto get_transaction =
-        [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
-          if (!commit_timestamp_and_accessor) {
-            commit_timestamp_and_accessor.emplace(commit_timestamp, Access());
-          } else if (commit_timestamp_and_accessor->first != commit_timestamp) {
-            throw utils::BasicException("Received more than one transaction!");
-          }
-          return &commit_timestamp_and_accessor->second;
-        };
-
-    bool transaction_complete = false;
-    for (uint64_t i = 0; !transaction_complete; ++i) {
-      DLOG(INFO) << "  Delta " << i;
-      const auto [timestamp, delta] = read_delta();
-
-      switch (delta.type) {
-        case durability::WalDeltaData::Type::VERTEX_CREATE: {
-          DLOG(INFO) << "       Create vertex "
-                     << delta.vertex_create_delete.gid.AsUint();
-          auto transaction = get_transaction(timestamp);
-          transaction->CreateVertex(delta.vertex_create_delete.gid);
-          break;
-        }
-        case durability::WalDeltaData::Type::VERTEX_DELETE: {
-          DLOG(INFO) << "       Delete vertex "
-                     << delta.vertex_create_delete.gid.AsUint();
-          auto transaction = get_transaction(timestamp);
-          auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid,
-                                                storage::View::NEW);
-          if (!vertex) throw utils::BasicException("Invalid transaction!");
-          auto ret = transaction->DeleteVertex(&*vertex);
-          if (ret.HasError() || !ret.GetValue())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
-          DLOG(INFO) << "       Vertex "
-                     << delta.vertex_add_remove_label.gid.AsUint()
-                     << " add label " << delta.vertex_add_remove_label.label;
-          auto transaction = get_transaction(timestamp);
-          auto vertex = transaction->FindVertex(
-              delta.vertex_add_remove_label.gid, storage::View::NEW);
-          if (!vertex) throw utils::BasicException("Invalid transaction!");
-          auto ret = vertex->AddLabel(
-              transaction->NameToLabel(delta.vertex_add_remove_label.label));
-          if (ret.HasError() || !ret.GetValue())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
-          DLOG(INFO) << "       Vertex "
-                     << delta.vertex_add_remove_label.gid.AsUint()
-                     << " remove label " << delta.vertex_add_remove_label.label;
-          auto transaction = get_transaction(timestamp);
-          auto vertex = transaction->FindVertex(
-              delta.vertex_add_remove_label.gid, storage::View::NEW);
-          if (!vertex) throw utils::BasicException("Invalid transaction!");
-          auto ret = vertex->RemoveLabel(
-              transaction->NameToLabel(delta.vertex_add_remove_label.label));
-          if (ret.HasError() || !ret.GetValue())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
-          DLOG(INFO) << "       Vertex "
-                     << delta.vertex_edge_set_property.gid.AsUint()
-                     << " set property "
-                     << delta.vertex_edge_set_property.property << " to "
-                     << delta.vertex_edge_set_property.value;
-          auto transaction = get_transaction(timestamp);
-          auto vertex = transaction->FindVertex(
-              delta.vertex_edge_set_property.gid, storage::View::NEW);
-          if (!vertex) throw utils::BasicException("Invalid transaction!");
-          auto ret =
-              vertex->SetProperty(transaction->NameToProperty(
-                                      delta.vertex_edge_set_property.property),
-                                  delta.vertex_edge_set_property.value);
-          if (ret.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::EDGE_CREATE: {
-          DLOG(INFO) << "       Create edge "
-                     << delta.edge_create_delete.gid.AsUint() << " of type "
-                     << delta.edge_create_delete.edge_type << " from vertex "
-                     << delta.edge_create_delete.from_vertex.AsUint()
-                     << " to vertex "
-                     << delta.edge_create_delete.to_vertex.AsUint();
-          auto transaction = get_transaction(timestamp);
-          auto from_vertex = transaction->FindVertex(
-              delta.edge_create_delete.from_vertex, storage::View::NEW);
-          if (!from_vertex) throw utils::BasicException("Invalid transaction!");
-          auto to_vertex = transaction->FindVertex(
-              delta.edge_create_delete.to_vertex, storage::View::NEW);
-          if (!to_vertex) throw utils::BasicException("Invalid transaction!");
-          auto edge = transaction->CreateEdge(
-              &*from_vertex, &*to_vertex,
-              transaction->NameToEdgeType(delta.edge_create_delete.edge_type),
-              delta.edge_create_delete.gid);
-          if (edge.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::EDGE_DELETE: {
-          DLOG(INFO) << "       Delete edge "
-                     << delta.edge_create_delete.gid.AsUint() << " of type "
-                     << delta.edge_create_delete.edge_type << " from vertex "
-                     << delta.edge_create_delete.from_vertex.AsUint()
-                     << " to vertex "
-                     << delta.edge_create_delete.to_vertex.AsUint();
-          auto transaction = get_transaction(timestamp);
-          auto from_vertex = transaction->FindVertex(
-              delta.edge_create_delete.from_vertex, storage::View::NEW);
-          if (!from_vertex) throw utils::BasicException("Invalid transaction!");
-          auto to_vertex = transaction->FindVertex(
-              delta.edge_create_delete.to_vertex, storage::View::NEW);
-          if (!to_vertex) throw utils::BasicException("Invalid transaction!");
-          auto edges = from_vertex->OutEdges(
-              storage::View::NEW,
-              {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)},
-              &*to_vertex);
-          if (edges.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          if (edges->size() != 1)
-            throw utils::BasicException("Invalid transaction!");
-          auto &edge = (*edges)[0];
-          auto ret = transaction->DeleteEdge(&edge);
-          if (ret.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
-          DLOG(INFO) << "       Edge "
-                     << delta.vertex_edge_set_property.gid.AsUint()
-                     << " set property "
-                     << delta.vertex_edge_set_property.property << " to "
-                     << delta.vertex_edge_set_property.value;
-
-          if (!config_.items.properties_on_edges)
-            throw utils::BasicException(
-                "Can't set properties on edges because properties on edges "
-                "are disabled!");
-
-          auto transaction = get_transaction(timestamp);
-
-          // The following block of code effectively implements `FindEdge` and
-          // yields an accessor that is only valid for managing the edge's
-          // properties.
-          auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
-          if (edge == edge_acc.end())
-            throw utils::BasicException("Invalid transaction!");
-          // The edge visibility check must be done here manually because we
-          // don't allow direct access to the edges through the public API.
-          {
-            bool is_visible = true;
-            Delta *delta = nullptr;
-            {
-              std::lock_guard<utils::SpinLock> guard(edge->lock);
-              is_visible = !edge->deleted;
-              delta = edge->delta;
-            }
-            ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW,
-                               [&is_visible](const Delta &delta) {
-                                 switch (delta.action) {
-                                   case Delta::Action::ADD_LABEL:
-                                   case Delta::Action::REMOVE_LABEL:
-                                   case Delta::Action::SET_PROPERTY:
-                                   case Delta::Action::ADD_IN_EDGE:
-                                   case Delta::Action::ADD_OUT_EDGE:
-                                   case Delta::Action::REMOVE_IN_EDGE:
-                                   case Delta::Action::REMOVE_OUT_EDGE:
-                                     break;
-                                   case Delta::Action::RECREATE_OBJECT: {
-                                     is_visible = true;
-                                     break;
-                                   }
-                                   case Delta::Action::DELETE_OBJECT: {
-                                     is_visible = false;
-                                     break;
-                                   }
-                                 }
-                               });
-            if (!is_visible)
-              throw utils::BasicException("Invalid transaction!");
-          }
-          EdgeRef edge_ref(&*edge);
-          // Here we create an edge accessor that we will use to get the
-          // properties of the edge. The accessor is created with an invalid
-          // type and invalid from/to pointers because we don't know them
-          // here, but that isn't an issue because we won't use that part of
-          // the API here.
-          auto ea = EdgeAccessor{edge_ref,
-                                 EdgeTypeId::FromUint(0UL),
-                                 nullptr,
-                                 nullptr,
-                                 &transaction->transaction_,
-                                 &indices_,
-                                 &constraints_,
-                                 config_.items};
-
-          auto ret =
-              ea.SetProperty(transaction->NameToProperty(
-                                 delta.vertex_edge_set_property.property),
-                             delta.vertex_edge_set_property.value);
-          if (ret.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-
-        case durability::WalDeltaData::Type::TRANSACTION_END: {
-          DLOG(INFO) << "       Transaction end";
-          if (!commit_timestamp_and_accessor ||
-              commit_timestamp_and_accessor->first != timestamp)
-            throw utils::BasicException("Invalid data!");
-          auto ret = commit_timestamp_and_accessor->second.Commit(
-              commit_timestamp_and_accessor->first);
-          if (ret.HasError())
-            throw utils::BasicException("Invalid transaction!");
-          commit_timestamp_and_accessor = std::nullopt;
-          break;
-        }
-
-        case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
-          DLOG(INFO) << "       Create label index on :"
-                     << delta.operation_label.label;
-          // Need to send the timestamp
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          if (!CreateIndex(NameToLabel(delta.operation_label.label), timestamp))
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
-          DLOG(INFO) << "       Drop label index on :"
-                     << delta.operation_label.label;
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          if (!DropIndex(NameToLabel(delta.operation_label.label), timestamp))
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
-          DLOG(INFO) << "       Create label+property index on :"
-                     << delta.operation_label_property.label << " ("
-                     << delta.operation_label_property.property << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          if (!CreateIndex(
-                  NameToLabel(delta.operation_label_property.label),
-                  NameToProperty(delta.operation_label_property.property),
-                  timestamp))
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
-          DLOG(INFO) << "       Drop label+property index on :"
-                     << delta.operation_label_property.label << " ("
-                     << delta.operation_label_property.property << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          if (!DropIndex(
-                  NameToLabel(delta.operation_label_property.label),
-                  NameToProperty(delta.operation_label_property.property),
-                  timestamp))
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
-          DLOG(INFO) << "       Create existence constraint on :"
-                     << delta.operation_label_property.label << " ("
-                     << delta.operation_label_property.property << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          auto ret = CreateExistenceConstraint(
-              NameToLabel(delta.operation_label_property.label),
-              NameToProperty(delta.operation_label_property.property),
-              timestamp);
-          if (!ret.HasValue() || !ret.GetValue())
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
-          DLOG(INFO) << "       Drop existence constraint on :"
-                     << delta.operation_label_property.label << " ("
-                     << delta.operation_label_property.property << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          if (!DropExistenceConstraint(
-                  NameToLabel(delta.operation_label_property.label),
-                  NameToProperty(delta.operation_label_property.property),
-                  timestamp))
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
-          std::stringstream ss;
-          utils::PrintIterable(ss, delta.operation_label_properties.properties);
-          DLOG(INFO) << "       Create unique constraint on :"
-                     << delta.operation_label_properties.label << " ("
-                     << ss.str() << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          std::set<PropertyId> properties;
-          for (const auto &prop : delta.operation_label_properties.properties) {
-            properties.emplace(NameToProperty(prop));
-          }
-          auto ret = CreateUniqueConstraint(
-              NameToLabel(delta.operation_label_properties.label), properties,
-              timestamp);
-          if (!ret.HasValue() ||
-              ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-        case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
-          std::stringstream ss;
-          utils::PrintIterable(ss, delta.operation_label_properties.properties);
-          DLOG(INFO) << "       Drop unique constraint on :"
-                     << delta.operation_label_properties.label << " ("
-                     << ss.str() << ")";
-          if (commit_timestamp_and_accessor)
-            throw utils::BasicException("Invalid transaction!");
-          std::set<PropertyId> properties;
-          for (const auto &prop : delta.operation_label_properties.properties) {
-            properties.emplace(NameToProperty(prop));
-          }
-          auto ret = DropUniqueConstraint(
-              NameToLabel(delta.operation_label_properties.label), properties,
-              timestamp);
-          if (ret != UniqueConstraints::DeletionStatus::SUCCESS)
-            throw utils::BasicException("Invalid transaction!");
-          break;
-        }
-      }
-      transaction_complete = is_transaction_complete(delta.type);
-    }
-
-    if (commit_timestamp_and_accessor)
-      throw utils::BasicException("Invalid data!");
-
-    AppendDeltasRes res{true, last_commit_timestamp_.load()};
-    slk::Save(res, res_builder);
-  });
-  replication_server_->rpc_server->Register<SnapshotRpc>(
-      [this](auto *req_reader, auto *res_builder) {
-        DLOG(INFO) << "Received SnapshotRpc";
-        SnapshotReq req;
-        slk::Load(&req, req_reader);
-
-        replication::Decoder decoder(req_reader);
-
-        utils::EnsureDirOrDie(snapshot_directory_);
-
-        const auto maybe_snapshot_path = decoder.ReadFile(snapshot_directory_);
-        CHECK(maybe_snapshot_path) << "Failed to load snapshot!";
-        DLOG(INFO) << "Received snapshot saved to " << *maybe_snapshot_path;
-
-        std::unique_lock<utils::RWLock> storage_guard(main_lock_);
-        // Clear the database
-        vertices_.clear();
-        edges_.clear();
-
-        constraints_ = Constraints();
-        // TODO (antonio2368): Check if there's a less hacky way
-        indices_.label_index =
-            LabelIndex(&indices_, &constraints_, config_.items);
-        indices_.label_property_index =
-            LabelPropertyIndex(&indices_, &constraints_, config_.items);
-        try {
-          DLOG(INFO) << "Loading snapshot";
-          auto recovered_snapshot = durability::LoadSnapshot(
-              *maybe_snapshot_path, &vertices_, &edges_, &name_id_mapper_,
-              &edge_count_, config_.items);
-          DLOG(INFO) << "Snapshot loaded successfully";
-          // If this step is present it should always be the first step of
-          // the recovery so we use the UUID we read from snasphost
-          uuid_ = recovered_snapshot.snapshot_info.uuid;
-          const auto &recovery_info = recovered_snapshot.recovery_info;
-          vertex_id_ = recovery_info.next_vertex_id;
-          edge_id_ = recovery_info.next_edge_id;
-          timestamp_ = std::max(timestamp_, recovery_info.next_timestamp);
-
-          durability::RecoverIndicesAndConstraints(
-              recovered_snapshot.indices_constraints, &indices_, &constraints_,
-              &vertices_);
-        } catch (const durability::RecoveryFailure &e) {
-          // TODO (antonio2368): What to do if the sent snapshot is invalid
-          LOG(WARNING) << "Couldn't load the snapshot because of: " << e.what();
-        }
-        last_commit_timestamp_ = timestamp_ - 1;
-        storage_guard.unlock();
-
-        SnapshotRes res{true, last_commit_timestamp_.load()};
-        slk::Save(res, res_builder);
-
-        // Delete other durability files
-        auto snapshot_files =
-            durability::GetSnapshotFiles(snapshot_directory_, uuid_);
-        for (const auto &[path, uuid, _] : snapshot_files) {
-          if (path != *maybe_snapshot_path) {
-            file_retainer_.DeleteFile(path);
-          }
-        }
-
-        auto wal_files = durability::GetWalFiles(wal_directory_, uuid_);
-        if (wal_files) {
-          for (const auto &[seq_num, from_timestamp, to_timestamp, _, path] :
-               *wal_files) {
-            file_retainer_.DeleteFile(path);
-          }
-
-          wal_file_.reset();
-        }
-      });
-  replication_server_->rpc_server->Register<OnlySnapshotRpc>(
-      [this](auto *req_reader, auto *res_builder) {
-        DLOG(INFO) << "Received OnlySnapshotRpc";
-        OnlySnapshotReq req;
-        slk::Load(&req, req_reader);
-
-        CHECK(last_commit_timestamp_.load() < req.snapshot_timestamp)
-            << "Invalid snapshot timestamp, it should be less than the last"
-               "commited timestamp";
-
-        last_commit_timestamp_.store(req.snapshot_timestamp);
-
-        SnapshotRes res{true, last_commit_timestamp_.load()};
-        slk::Save(res, res_builder);
-      });
-  replication_server_->rpc_server->Register<WalFilesRpc>(
-      [this](auto *req_reader, auto *res_builder) {
-        DLOG(INFO) << "Received WalFilesRpc";
-        WalFilesReq req;
-        slk::Load(&req, req_reader);
-
-        const auto wal_file_number = req.file_number;
-        DLOG(INFO) << "Received WAL files: " << wal_file_number;
-
-        replication::Decoder decoder(req_reader);
-
-        utils::EnsureDirOrDie(wal_directory_);
-
-        std::unique_lock<utils::RWLock> storage_guard(main_lock_);
-        durability::RecoveredIndicesAndConstraints indices_constraints;
-        auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
-        if (wal_info.seq_num == 0) {
-          uuid_ = wal_info.uuid;
-        }
-        // Check the seq number of the first wal file to see if it's the
-        // finalized form of the current wal on replica
-        if (wal_file_) {
-          if (wal_file_->SequenceNumber() == wal_info.seq_num &&
-              wal_file_->Path() != path) {
-            wal_file_->DeleteWal();
-          }
-          wal_file_.reset();
-        }
-
-        for (auto i = 1; i < wal_file_number; ++i) {
-          LoadWal(&decoder, &indices_constraints);
-        }
-
-        durability::RecoverIndicesAndConstraints(indices_constraints, &indices_,
-                                                 &constraints_, &vertices_);
-        storage_guard.unlock();
-
-        WalFilesRes res{true, last_commit_timestamp_.load()};
-        slk::Save(res, res_builder);
-      });
-  replication_server_->rpc_server->Register<CurrentWalRpc>(
-      [this](auto *req_reader, auto *res_builder) {
-        DLOG(INFO) << "Received CurrentWalRpc";
-        CurrentWalReq req;
-        slk::Load(&req, req_reader);
-
-        replication::Decoder decoder(req_reader);
-
-        utils::EnsureDirOrDie(wal_directory_);
-
-        std::unique_lock<utils::RWLock> storage_guard(main_lock_);
-        durability::RecoveredIndicesAndConstraints indices_constraints;
-        auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
-        if (wal_info.seq_num == 0) {
-          uuid_ = wal_info.uuid;
-        }
-
-        if (wal_file_ && wal_file_->SequenceNumber() == wal_info.seq_num &&
-            wal_file_->Path() != path) {
-          // Delete the old wal file
-          file_retainer_.DeleteFile(wal_file_->Path());
-        }
-        CHECK(config_.durability.snapshot_wal_mode ==
-              Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL);
-        wal_file_.emplace(std::move(path), config_.items, &name_id_mapper_,
-                          wal_info.seq_num, wal_info.from_timestamp,
-                          wal_info.to_timestamp, wal_info.num_deltas,
-                          &file_retainer_);
-        durability::RecoverIndicesAndConstraints(indices_constraints, &indices_,
-                                                 &constraints_, &vertices_);
-        storage_guard.unlock();
-
-        CurrentWalRes res{true, last_commit_timestamp_.load()};
-        slk::Save(res, res_builder);
-      });
-  replication_server_->rpc_server->Start();
+  // Generate new epoch id and save the last one to the history.
+  if (epoch_history_.size() == kEpochHistoryRetention) {
+    epoch_history_.pop_front();
+  }
+  epoch_history_.emplace_back(std::move(epoch_id_), last_commit_timestamp_);
+  epoch_id_ = utils::GenerateUUID();
 }
 
 void Storage::RegisterReplica(
     std::string name, io::network::Endpoint endpoint,
     const replication::ReplicationMode replication_mode) {
-  std::shared_lock guard(replication_lock_);
   // TODO (antonio2368): This shouldn't stop the main instance
   CHECK(replication_role_.load() == ReplicationRole::MAIN)
       << "Only main instance can register a replica!";
 
-  // We can safely add new elements to the list because it doesn't validate
-  // existing references/iteratos
   replication_clients_.WithLock([&](auto &clients) {
     if (std::any_of(clients.begin(), clients.end(),
-                    [&](auto &client) { return client.Name() == name; })) {
+                    [&](auto &client) { return client->Name() == name; })) {
       throw utils::BasicException("Replica with a same name already exists!");
     }
-    clients.emplace_back(std::move(name), last_commit_timestamp_,
-                         &name_id_mapper_, config_.items, &file_retainer_,
-                         snapshot_directory_, wal_directory_, uuid_, &wal_file_,
-                         &engine_lock_, endpoint, false, replication_mode);
   });
+
+  auto client = std::make_unique<ReplicationClient>(
+      std::move(name), this, endpoint, false, replication_mode);
+
+  replication_clients_.WithLock(
+      [&](auto &clients) { clients.push_back(std::move(client)); });
 }
 
 void Storage::UnregisterReplica(const std::string_view name) {
-  std::unique_lock<utils::RWLock> replication_guard(replication_lock_);
   CHECK(replication_role_.load() == ReplicationRole::MAIN)
       << "Only main instance can unregister a replica!";
   replication_clients_.WithLock([&](auto &clients) {
-    clients.remove_if(
-        [&](const auto &client) { return client.Name() == name; });
+    std::erase_if(clients,
+                  [&](const auto &client) { return client->Name() == name; });
   });
 }
 
-std::optional<replication::ReplicaState> Storage::ReplicaState(
+std::optional<replication::ReplicaState> Storage::GetReplicaState(
     const std::string_view name) {
   return replication_clients_.WithLock(
       [&](auto &clients) -> std::optional<replication::ReplicaState> {
         const auto client_it = std::find_if(
             clients.cbegin(), clients.cend(),
-            [name](auto &client) { return client.Name() == name; });
+            [name](auto &client) { return client->Name() == name; });
         if (client_it == clients.cend()) {
           return std::nullopt;
         }
-        return client_it->State();
+        return (*client_it)->State();
       });
 }
 #endif

src/storage/v2/storage.hpp

@@ -24,10 +24,11 @@
 #include "utils/scheduler.hpp"
 #include "utils/skip_list.hpp"
 #include "utils/synchronized.hpp"
+#include "utils/uuid.hpp"
 
 #ifdef MG_ENTERPRISE
 #include "rpc/server.hpp"
-#include "storage/v2/replication/replication.hpp"
+#include "storage/v2/replication/enums.hpp"
 #include "storage/v2/replication/rpc.hpp"
 #include "storage/v2/replication/serialization.hpp"
 #endif
@@ -419,13 +420,10 @@ class Storage final {
       return;
     }
 
-    std::unique_lock<utils::RWLock> replication_guard(replication_lock_);
-
     if constexpr (role == ReplicationRole::REPLICA) {
       ConfigureReplica(std::forward<Args>(args)...);
-    } else if (role == ReplicationRole::MAIN) {
-      // Main instance does not need replication server
-      replication_server_.reset();
+    } else if constexpr (role == ReplicationRole::MAIN) {
+      ConfigureMain(std::forward<Args>(args)...);
     }
 
     replication_role_.store(role);
@@ -436,7 +434,8 @@ class Storage final {
                            replication::ReplicationMode::SYNC);
   void UnregisterReplica(std::string_view name);
 
-  std::optional<replication::ReplicaState> ReplicaState(std::string_view name);
+  std::optional<replication::ReplicaState> GetReplicaState(
+      std::string_view name);
 #endif
 
  private:
@@ -462,10 +461,7 @@ class Storage final {
 
 #ifdef MG_ENTERPRISE
   void ConfigureReplica(io::network::Endpoint endpoint);
-
-  std::pair<durability::WalInfo, std::filesystem::path> LoadWal(
-      replication::Decoder *decoder,
-      durability::RecoveredIndicesAndConstraints *indices_constraints);
+  void ConfigureMain();
 #endif
 
   // Main storage lock.
@@ -538,6 +534,26 @@ class Storage final {
   // Sequence number used to keep track of the chain of WALs.
   uint64_t wal_seq_num_{0};
 
+  // UUID to distinguish different main instance runs for replication process
+  // on SAME storage.
+  // Multiple instances can have same storage UUID and be MAIN at the same time.
+  // We cannot compare commit timestamps of those instances if one of them
+  // becomes the replica of the other so we use epoch_id_ as additional
+  // discriminating property.
+  // Example of this:
+  // We have 2 instances of the same storage, S1 and S2.
+  // S1 and S2 are MAIN and accept their own commits and write them to the WAL.
+  // At the moment when S1 commited a transaction with timestamp 20, and S2
+  // a different transaction with timestamp 15, we change S2's role to REPLICA
+  // and register it on S1.
+  // Without using the epoch_id, we don't know that S1 and S2 have completely
+  // different transactions, we think that the S2 is behind only by 5 commits.
+  std::string epoch_id_;
+  // History of the previous epoch ids.
+  // Each value consists of the epoch id along the last commit belonging to that
+  // epoch.
+  std::deque<std::pair<std::string, uint64_t>> epoch_history_;
+
   std::optional<durability::WalFile> wal_file_;
   uint64_t wal_unsynced_transactions_{0};
 
@@ -545,32 +561,26 @@ class Storage final {
 
   // Replication
 #ifdef MG_ENTERPRISE
+  // Last commited timestamp
   std::atomic<uint64_t> last_commit_timestamp_{kTimestampInitialId};
-  utils::RWLock replication_lock_{utils::RWLock::Priority::WRITE};
 
-  struct ReplicationServer {
-    std::optional<communication::ServerContext> rpc_server_context;
-    std::optional<rpc::Server> rpc_server;
-
-    explicit ReplicationServer() = default;
-    ReplicationServer(const ReplicationServer &) = delete;
-    ReplicationServer(ReplicationServer &&) = delete;
-    ReplicationServer &operator=(const ReplicationServer &) = delete;
-    ReplicationServer &operator=(ReplicationServer &&) = delete;
-
-    ~ReplicationServer() {
-      if (rpc_server) {
-        rpc_server->Shutdown();
-        rpc_server->AwaitShutdown();
-      }
-    }
-  };
+  class ReplicationServer;
+  std::unique_ptr<ReplicationServer> replication_server_{nullptr};
 
+  class ReplicationClient;
+  // We create ReplicationClient using unique_ptr so we can move
+  // newly created client into the vector.
+  // We cannot move the client directly because it contains ThreadPool
+  // which cannot be moved. Also, the move is necessary because
+  // we don't want to create the client directly inside the vector
+  // because that would require the lock on the list putting all
+  // commits (they iterate list of clients) to halt.
+  // This way we can initiliaze client in main thread which means
+  // that we can immediately notify the user if the intiialization
+  // failed.
   using ReplicationClientList =
-      utils::Synchronized<std::list<replication::ReplicationClient>,
+      utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>,
                           utils::SpinLock>;
-
-  std::optional<ReplicationServer> replication_server_;
   ReplicationClientList replication_clients_;
 
   std::atomic<ReplicationRole> replication_role_{ReplicationRole::MAIN};

tests/integration/durability/runner.py

@@ -15,7 +15,9 @@ TESTS_DIR = os.path.join(SCRIPT_DIR, "tests")
 
 SNAPSHOT_FILE_NAME = "snapshot.bin"
 WAL_FILE_NAME = "wal.bin"
-DUMP_FILE_NAME = "expected.cypher"
+
+DUMP_SNAPSHOT_FILE_NAME = "expected_snapshot.cypher"
+DUMP_WAL_FILE_NAME = "expected_wal.cypher"
 
 
 def wait_for_server(port, delay=0.1):
@@ -38,7 +40,12 @@ def list_to_string(data):
     return ret
 
 
-def execute_test(memgraph_binary, dump_binary, test_directory, test_type):
+def execute_test(
+        memgraph_binary,
+        dump_binary,
+        test_directory,
+        test_type,
+        write_expected):
     assert test_type in ["SNAPSHOT", "WAL"], \
         "Test type should be either 'SNAPSHOT' or 'WAL'."
     print("\033[1;36m~~ Executing test {} ({}) ~~\033[0m"
@@ -82,15 +89,25 @@ def execute_test(memgraph_binary, dump_binary, test_directory, test_type):
     memgraph.terminate()
     assert memgraph.wait() == 0, "Memgraph process didn't exit cleanly!"
 
-    # Compare dump files
-    expected_dump_file = os.path.join(test_directory, DUMP_FILE_NAME)
-    assert os.path.exists(expected_dump_file), \
-        "Could not find expected dump path {}".format(expected_dump_file)
-    queries_got = sorted_content(dump_output_file.name)
-    queries_expected = sorted_content(expected_dump_file)
-    assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
-        "{}".format(list_to_string(queries_got),
-                    list_to_string(queries_expected))
+    dump_file_name = DUMP_SNAPSHOT_FILE_NAME if test_type == "SNAPSHOT" else DUMP_WAL_FILE_NAME
+
+    if write_expected:
+        with open(dump_output_file.name, 'r') as dump:
+            queries_got = dump.readlines()
+        # Write dump files
+        expected_dump_file = os.path.join(test_directory, dump_file_name)
+        with open(expected_dump_file, 'w') as expected:
+            expected.writelines(queries_got)
+    else:
+        # Compare dump files
+        expected_dump_file = os.path.join(test_directory, dump_file_name)
+        assert os.path.exists(expected_dump_file), \
+            "Could not find expected dump path {}".format(expected_dump_file)
+        queries_got = sorted_content(dump_output_file.name)
+        queries_expected = sorted_content(expected_dump_file)
+        assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
+            "{}".format(list_to_string(queries_got),
+                        list_to_string(queries_expected))
 
     print("\033[1;32m~~ Test successful ~~\033[0m\n")
 
@@ -112,9 +129,11 @@ def find_test_directories(directory):
                 continue
             snapshot_file = os.path.join(test_dir_path, SNAPSHOT_FILE_NAME)
             wal_file = os.path.join(test_dir_path, WAL_FILE_NAME)
-            dump_file = os.path.join(test_dir_path, DUMP_FILE_NAME)
-            if (os.path.isfile(snapshot_file) and os.path.isfile(dump_file) and
-                    os.path.isfile(wal_file)):
+            dump_snapshot_file = os.path.join(
+                test_dir_path, DUMP_SNAPSHOT_FILE_NAME)
+            dump_wal_file = os.path.join(test_dir_path, DUMP_WAL_FILE_NAME)
+            if (os.path.isfile(snapshot_file) and os.path.isfile(dump_snapshot_file)
+                    and os.path.isfile(wal_file) and os.path.isfile(dump_wal_file)):
                 test_dirs.append(test_dir_path)
             else:
                 raise Exception("Missing data in test directory '{}'"
@@ -129,13 +148,27 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--memgraph", default=memgraph_binary)
     parser.add_argument("--dump", default=dump_binary)
+    parser.add_argument(
+        '--write-expected',
+        action='store_true',
+        help='Overwrite the expected cypher with results from current run')
     args = parser.parse_args()
 
     test_directories = find_test_directories(TESTS_DIR)
     assert len(test_directories) > 0, "No tests have been found!"
 
     for test_directory in test_directories:
-        execute_test(args.memgraph, args.dump, test_directory, "SNAPSHOT")
-        execute_test(args.memgraph, args.dump, test_directory, "WAL")
+        execute_test(
+            args.memgraph,
+            args.dump,
+            test_directory,
+            "SNAPSHOT",
+            args.write_expected)
+        execute_test(
+            args.memgraph,
+            args.dump,
+            test_directory,
+            "WAL",
+            args.write_expected)
 
     sys.exit(0)

tests/integration/durability/tests/v12/test_all/expected.cypher

@@ -1,11 +0,0 @@
-CREATE INDEX ON :`label`;
-CREATE INDEX ON :`label2`(`prop2`);
-CREATE INDEX ON :`label2`(`prop`);
-CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `prop`: "joj", `ext`: 2});
-CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `prop`: "joj", `ext`: 2});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_constraints/expected.cypher

@@ -1,10 +0,0 @@
-CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`prop`);
-CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`prop1`);
-CREATE CONSTRAINT ON (u:`labellabel`) ASSERT EXISTS (u.`prop`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label` {__mg_id__: 0, `prop`: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 1, `prop`: false});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop1`: 1});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop1`: 2});
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_edges/expected.cypher

@@ -1,45 +0,0 @@
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__ {__mg_id__: 0});
-CREATE (:__mg_vertex__ {__mg_id__: 1});
-CREATE (:__mg_vertex__ {__mg_id__: 2});
-CREATE (:__mg_vertex__ {__mg_id__: 3});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 4});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 5});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 6});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 7});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 8});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 9});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 10});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 11});
-CREATE (:__mg_vertex__ {__mg_id__: 12});
-CREATE (:__mg_vertex__ {__mg_id__: 13});
-CREATE (:__mg_vertex__ {__mg_id__: 14});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 12 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 14 CREATE (u)-[:`selfedge`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 11 CREATE (u)-[:`selfedge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 12 CREATE (u)-[:`selfedge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 13 CREATE (u)-[:`selfedge2`]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_edges_with_properties/expected.cypher

@@ -1,24 +0,0 @@
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__ {__mg_id__: 0});
-CREATE (:__mg_vertex__ {__mg_id__: 1});
-CREATE (:__mg_vertex__ {__mg_id__: 2});
-CREATE (:__mg_vertex__ {__mg_id__: 3});
-CREATE (:__mg_vertex__ {__mg_id__: 4});
-CREATE (:__mg_vertex__ {__mg_id__: 5});
-CREATE (:__mg_vertex__ {__mg_id__: 6});
-CREATE (:__mg_vertex__ {__mg_id__: 7});
-CREATE (:__mg_vertex__ {__mg_id__: 8});
-CREATE (:__mg_vertex__ {__mg_id__: 9});
-CREATE (:__mg_vertex__ {__mg_id__: 10});
-CREATE (:__mg_vertex__ {__mg_id__: 11});
-CREATE (:__mg_vertex__ {__mg_id__: 12});
-CREATE (:__mg_vertex__ {__mg_id__: 13});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 1}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: false}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: 1, `prop2`: {`prop4`: 9}}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, ""]}]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_indices/expected.cypher

@@ -1,16 +0,0 @@
-CREATE INDEX ON :`label2`;
-CREATE INDEX ON :`label1`;
-CREATE INDEX ON :`label3`;
-CREATE INDEX ON :`label`(`prop`);
-CREATE INDEX ON :`label2`(`prop`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label` {__mg_id__: 0});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 2});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 4, `prop`: 2});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop`: 3});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 6, `prop2`: 1});
-CREATE (:__mg_vertex__:`label3` {__mg_id__: 7});
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v14/test_all/expected_snapshot.cypher

@@ -0,0 +1,16 @@
+CREATE INDEX ON :`label`;
+CREATE INDEX ON :`label2`(`prop2`);
+CREATE INDEX ON :`label2`(`prop`);
+CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
+CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE;
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"});
+CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `ext`: 2, `prop`: "joj"});
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1});
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v14/test_constraints/expected_snapshot.cypher

@@ -0,0 +1,6 @@
+CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`);
+CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
+CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE;
+CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE;
+CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE;
+CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`b`, u.`a` IS UNIQUE;

tests/integration/durability/tests/v14/test_edges/expected_wal.cypher

@@ -0,0 +1,58 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__ {__mg_id__: 0});
+CREATE (:__mg_vertex__ {__mg_id__: 1});
+CREATE (:__mg_vertex__ {__mg_id__: 2});
+CREATE (:__mg_vertex__ {__mg_id__: 3});
+CREATE (:__mg_vertex__ {__mg_id__: 4});
+CREATE (:__mg_vertex__ {__mg_id__: 5});
+CREATE (:__mg_vertex__ {__mg_id__: 6});
+CREATE (:__mg_vertex__ {__mg_id__: 7});
+CREATE (:__mg_vertex__ {__mg_id__: 8});
+CREATE (:__mg_vertex__ {__mg_id__: 9});
+CREATE (:__mg_vertex__ {__mg_id__: 10});
+CREATE (:__mg_vertex__ {__mg_id__: 11});
+CREATE (:__mg_vertex__ {__mg_id__: 12});
+CREATE (:__mg_vertex__ {__mg_id__: 13});
+CREATE (:__mg_vertex__:`label` {__mg_id__: 14});
+CREATE (:__mg_vertex__:`label` {__mg_id__: 15});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v14/test_indices/expected_wal.cypher

@@ -0,0 +1,4 @@
+CREATE INDEX ON :`label2`;
+CREATE INDEX ON :`label2`(`prop2`);
+CREATE INDEX ON :`label`(`prop2`);
+CREATE INDEX ON :`label`(`prop`);

tests/integration/durability/tests/v14/test_vertices/expected_snapshot.cypher

@@ -5,9 +5,9 @@ CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true});
 CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141});
-CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop2`: -314000000, `prop3`: true});
-CREATE (:__mg_vertex__:`label1`:`label2`:`label3` {__mg_id__: 7});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop`: 1, `prop2`: 2, `prop3`: "str"});
+CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop3`: true, `prop2`: -314000000});
+CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7});
+CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop3`: "str", `prop2`: 2, `prop`: 1});
 CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]});
 CREATE (:__mg_vertex__:`label`:`label3` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}});

tests/integration/durability/tests/v14/test_vertices/expected_wal.cypher

@@ -6,7 +6,7 @@ CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true});
 CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141});
 CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop2`: -314000000, `prop3`: true});
-CREATE (:__mg_vertex__:`label1`:`label2`:`label3` {__mg_id__: 7});
+CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop`: 1, `prop2`: 2, `prop3`: "str"});
 CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}});
 CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]});

tests/integration/mg_import_csv/runner.py

@@ -75,7 +75,7 @@ def verify_lifetime(memgraph_binary, mg_import_csv_binary):
 
 
 def execute_test(name, test_path, test_config, memgraph_binary,
-                 mg_import_csv_binary, tester_binary):
+                 mg_import_csv_binary, tester_binary, write_expected):
     print("\033[1;36m~~ Executing test", name, "~~\033[0m")
     storage_directory = tempfile.TemporaryDirectory()
 
@@ -87,14 +87,8 @@ def execute_test(name, test_path, test_config, memgraph_binary,
         raise Exception("The test should specify either 'import_should_fail' "
                         "or 'expected'!")
 
-    # Load test expected queries
-    import_should_fail = test_config.pop("import_should_fail", False)
     expected_path = test_config.pop("expected", "")
-    if expected_path:
-        with open(os.path.join(test_path, expected_path)) as f:
-            queries_expected = extract_rows(f.read())
-    else:
-        queries_expected = ""
+    import_should_fail = test_config.pop("import_should_fail", False)
 
     # Generate common args
     properties_on_edges = bool(test_config.pop("properties_on_edges", False))
@@ -106,10 +100,10 @@ def execute_test(name, test_path, test_config, memgraph_binary,
     mg_import_csv_args = [mg_import_csv_binary] + common_args
     for key, value in test_config.items():
         flag = "--" + key.replace("_", "-")
-        if type(value) == list:
+        if isinstance(value, list):
             for item in value:
                 mg_import_csv_args.extend([flag, str(item)])
-        elif type(value) == bool:
+        elif isinstance(value, bool):
             mg_import_csv_args.append(flag + "=" + str(value).lower())
         else:
             mg_import_csv_args.extend([flag, str(value)])
@@ -117,7 +111,6 @@ def execute_test(name, test_path, test_config, memgraph_binary,
     # Execute mg_import_csv
     ret = subprocess.run(mg_import_csv_args, cwd=test_path)
 
-    # Check the return code
     if import_should_fail:
         if ret.returncode == 0:
             raise Exception("The import should have failed, but it "
@@ -154,12 +147,23 @@ def execute_test(name, test_path, test_config, memgraph_binary,
     memgraph.terminate()
     assert memgraph.wait() == 0, "Memgraph process didn't exit cleanly!"
 
-    # Verify the queries
-    queries_expected.sort()
-    queries_got.sort()
-    assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
-        "{}".format(list_to_string(queries_got),
-                    list_to_string(queries_expected))
+    if write_expected:
+        with open(os.path.join(test_path, expected_path), 'w') as expected:
+            expected.write('\n'.join(queries_got))
+
+    else:
+        if expected_path:
+            with open(os.path.join(test_path, expected_path)) as f:
+                queries_expected = extract_rows(f.read())
+        else:
+            queries_expected = ""
+
+        # Verify the queries
+        queries_expected.sort()
+        queries_got.sort()
+        assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
+            "{}".format(list_to_string(queries_got),
+                        list_to_string(queries_expected))
     print("\033[1;32m~~ Test successful ~~\033[0m\n")
 
 
@@ -174,6 +178,10 @@ if __name__ == "__main__":
     parser.add_argument("--memgraph", default=memgraph_binary)
     parser.add_argument("--mg-import-csv", default=mg_import_csv_binary)
     parser.add_argument("--tester", default=tester_binary)
+    parser.add_argument(
+        "--write-expected",
+        action="store_true",
+        help="Overwrite the expected values with the results of the current run")
     args = parser.parse_args()
 
     # First test whether the CSV importer can be started while the main
@@ -192,6 +200,6 @@ if __name__ == "__main__":
         for test_config in testcases:
             test_name = name + "/" + test_config.pop("name")
             execute_test(test_name, test_path, test_config, args.memgraph,
-                         args.mg_import_csv, args.tester)
+                         args.mg_import_csv, args.tester, args.write_expected)
 
     sys.exit(0)

tests/integration/mg_import_csv/tests/array_types/expected.cypher

@@ -7,4 +7,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment`:`Entry4`:`this`:`is`:`a`:` lot `:`of`
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`KNOWS` {`value`: ["5", " asd", " helloworld"]}]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:`KNOWS` {`value`: ["6", "hmmm"]}]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/array_types_multi_char_array_delimiter/expected.cypher

@@ -7,4 +7,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment`:`Entry4`:`this`:`is`:`a`:` lot `:`of`
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`KNOWS` {`value`: ["5", " asd", " helloworld"]}]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:`KNOWS` {`value`: ["6", "hmmm"]}]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/bad_relationships/expected.cypher

@@ -8,4 +8,4 @@ MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 2 CREATE (u)-[:`VISITED`]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`FOLLOWS`]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test1/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `id`: "0"});
 CREATE (:__mg_vertex__ {__mg_id__: 1, `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test10/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "world fgh", `id`:
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "will this work?", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test12/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "ÖworldÖÖÖ fÖ
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wilÖl this work?Ö", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test13/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "Öworld,Ö,ÖÖ,
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wilÖl t,his work?Ö", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test15/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "woÖrld", `id`: "
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wilÖl this work?ÖÖ", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test16/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `id`: "0"});
 CREATE (:__mg_vertex__ {__mg_id__: 1, `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test17/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "world", `id`: "0"
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "my", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test18/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "world fgh", `id`:
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "will this work?", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test2/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "world", `id`: "0"
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "my", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test20/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "\"world\"\"\" f\"
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l this work?\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test21/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "\"worldÖ\"Ö\"\"
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l tÖhis work?\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test23/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "wo\"rld", `id`: "
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l this work?\"\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test3/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "world fgh", `id`:
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "will this work?", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test5/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "\"world\"\"\" f\"
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l this work?\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test6/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "\"world,\",\"\",
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l t,his work?\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test8/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `test`: "asd", `value`: "wo\"rld", `id`: "
 CREATE (:__mg_vertex__ {__mg_id__: 1, `test`: "string", `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `test`: "wil\"l this work?\"\"", `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/csv_parser_test9/expected.cypher

@@ -3,4 +3,4 @@ CREATE (:__mg_vertex__ {__mg_id__: 0, `id`: "0"});
 CREATE (:__mg_vertex__ {__mg_id__: 1, `id`: "1"});
 CREATE (:__mg_vertex__ {__mg_id__: 2, `value`: "hello", `id`: "2"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/expected.cypher

@@ -14,4 +14,4 @@ MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 6 CREATE (u)-[:`NOT_NEIGHBOUR`]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 0 CREATE (u)-[:`NOT_NEIGHBOUR`]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/duplicate_nodes/expected.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `cou
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: "3"});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: "4"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/extra_columns_nodes/expected.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `cou
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: "3"});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: "4"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/extra_columns_relationships/expected.cypher

@@ -8,4 +8,4 @@ MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 2 CREATE (u)-[:`VISITED`]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`FOLLOWS`]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/field_types/expected.cypher

@@ -1,4 +1,4 @@
 CREATE INDEX ON :__mg_vertex__(__mg_id__);
 CREATE (:__mg_vertex__ {__mg_id__: 0, `value_bool`: false, `value_boolean`: true, `value_integer`: 5, `value_float`: 2.718, `value_double`: 3.141, `value_short`: 4, `value_byte`: 3, `value_long`: 2, `value_int`: 1, `value_char`: "world", `value_str`: "hello", `id`: "0"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/id_type/expected_integer.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `cou
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: 3});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: 4});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/id_type/expected_string.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `cou
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: "3"});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: "4"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignore_empty_strings/expected_disabled.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `bro
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: "3"});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: "4"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignore_empty_strings/expected_enabled.cypher

@@ -5,4 +5,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 2, `content`: "LOL", `cou
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 3, `content`: "I see", `browser`: "Firefox", `country`: "France", `id`: "3"});
 CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `browser`: "Internet Explorer", `country`: "Italy", `id`: "4"});
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignored_columns/expected.cypher

@@ -7,4 +7,4 @@ CREATE (:__mg_vertex__:`Message`:`Comment` {__mg_id__: 4, `content`: "fine", `br
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`KNOWS` {`value`: 5}]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:`KNOWS` {`value`: 6}]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/multi_char_quote_and_node_label/expected.cypher

@@ -15,4 +15,4 @@ MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 3 AND v.__mg_id__ = 8 CREATE (u)-[:`POSTED_ON`]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 9 CREATE (u)-[:`POSTED_ON`]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/neo_example/expected.cypher

@@ -15,4 +15,4 @@ MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`ACTED_IN` {`role`: "Trinity"}]->(v);
 MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 2 CREATE (u)-[:`ACTED_IN` {`role`: "Trinity"}]->(v);
 DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;