memgraph/tests/unit/database_key_index.cpp
florijan 1e0ac8ab8f Write-ahead log
Summary:
My dear fellow Memgraphians. It's friday afternoon, and I am as ready to pop as WAL is to get reviewed...

What's done:
- Vertices and Edges have global IDs, stored in `VersionList`. Main storage is now a concurrent map ID->vlist_ptr.
- WriteAheadLog class added. It's based around buffering WAL::Op objects (elementraly DB changes) and periodically serializing and flusing them to disk.
- Snapshot recovery refactored, WAL recovery added. Snapshot format changed again to include necessary info.
- Durability testing completely reworked.

What's not done (and should be when we decide how):
- Old WAL file purging.
- Config refactor (naming and organization). Will do when we discuss what we want.
- Changelog and new feature documentation (both depending on the point above).
- Better error handling and recovery feedback. Currently it's all returning bools, which is not fine-grained enough (neither for errors nor partial successes, also EOF is reported as a failure at the moment).
- Moving the implementation of WAL stuff to .cpp where possible.
- Not sure if there are transactions being created outside of `GraphDbAccessor` and it's `BuildIndex`. Need to look into.
- True write-ahead logic (flag controlled): not committing a DB transaction if the WAL has not flushed it's data. We can discuss the gain/effort ratio for this feature.

Reviewers: buda, mislav.bradac, teon.banek, dgleich

Reviewed By: dgleich

Subscribers: mtomic, pullbot

Differential Revision: https://phabricator.memgraph.io/D958
2017-11-13 09:51:39 +01:00

208 lines
6.2 KiB
C++

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "database/graph_db_accessor.hpp"
#include "database/graph_db_datatypes.hpp"
#include "storage/vertex.hpp"
#include "mvcc_gc_common.hpp"
using testing::UnorderedElementsAreArray;
// Test index does it insert everything uniquely
TEST(LabelsIndex, UniqueInsert) {
KeyIndex<GraphDbTypes::Label, Vertex> index;
GraphDb db;
GraphDbAccessor dba(db);
tx::Engine engine;
auto t1 = engine.Begin();
mvcc::VersionList<Vertex> vlist(*t1, 0);
t1->Commit();
auto t2 = engine.Begin();
vlist.find(*t2)->labels_.push_back(dba.Label("1"));
index.Update(dba.Label("1"), &vlist, vlist.find(*t2));
// Try multiple inserts
index.Update(dba.Label("1"), &vlist, vlist.find(*t2));
vlist.find(*t2)->labels_.push_back(dba.Label("2"));
index.Update(dba.Label("2"), &vlist, vlist.find(*t2));
vlist.find(*t2)->labels_.push_back(dba.Label("3"));
index.Update(dba.Label("3"), &vlist, vlist.find(*t2));
t2->Commit();
EXPECT_EQ(index.Count(dba.Label("1")), 1);
EXPECT_EQ(index.Count(dba.Label("2")), 1);
EXPECT_EQ(index.Count(dba.Label("3")), 1);
}
// Check if index filters duplicates.
TEST(LabelsIndex, UniqueFilter) {
GraphDb db;
KeyIndex<GraphDbTypes::Label, Vertex> index;
GraphDbAccessor dba(db);
tx::Engine engine;
auto t1 = engine.Begin();
mvcc::VersionList<Vertex> vlist1(*t1, 0);
mvcc::VersionList<Vertex> vlist2(*t1, 1);
engine.Advance(t1->id_);
auto r1v1 = vlist1.find(*t1);
auto r1v2 = vlist2.find(*t1);
EXPECT_NE(vlist1.find(*t1), nullptr);
auto label1 = dba.Label("1");
vlist1.find(*t1)->labels_.push_back(label1);
vlist2.find(*t1)->labels_.push_back(label1);
index.Update(label1, &vlist1, r1v1);
index.Update(label1, &vlist2, r1v2);
t1->Commit();
auto t2 = engine.Begin();
auto r2v1 = vlist1.update(*t2);
auto r2v2 = vlist2.update(*t2);
index.Update(label1, &vlist1, r2v1);
index.Update(label1, &vlist2, r2v2);
t2->Commit();
auto t3 = engine.Begin();
std::vector<mvcc::VersionList<Vertex> *> expected = {&vlist1, &vlist2};
sort(expected.begin(),
expected.end()); // Entries will be sorted by vlist pointers.
int cnt = 0;
for (auto vlist : index.GetVlists(label1, *t3, false)) {
EXPECT_LT(cnt, expected.size());
EXPECT_EQ(vlist, expected[cnt++]);
}
}
// Delete not anymore relevant recods from index.
TEST(LabelsIndex, Refresh) {
KeyIndex<GraphDbTypes::Label, Vertex> index;
GraphDb db;
GraphDbAccessor access(db);
tx::Engine engine;
// add two vertices to database
auto t1 = engine.Begin();
mvcc::VersionList<Vertex> vlist1(*t1, 0);
mvcc::VersionList<Vertex> vlist2(*t1, 1);
engine.Advance(t1->id_);
auto v1r1 = vlist1.find(*t1);
auto v2r1 = vlist2.find(*t1);
EXPECT_NE(v1r1, nullptr);
EXPECT_NE(v2r1, nullptr);
auto label = access.Label("label");
v1r1->labels_.push_back(label);
v2r1->labels_.push_back(label);
index.Update(label, &vlist1, v1r1);
index.Update(label, &vlist2, v2r1);
t1->Commit();
auto t2 = engine.Begin();
auto v1r2 = vlist1.update(*t2);
auto v2r2 = vlist2.update(*t2);
index.Update(label, &vlist1, v1r2);
index.Update(label, &vlist2, v2r2);
index.Refresh(GcSnapshot(engine, t2), engine);
EXPECT_EQ(index.Count(label), 4);
t2->Commit();
EXPECT_EQ(index.Count(label), 4);
index.Refresh(GcSnapshot(engine, nullptr), engine);
EXPECT_EQ(index.Count(label), 2);
}
// Transaction hasn't ended and so the vertex is not visible.
TEST(LabelsIndexDb, AddGetZeroLabels) {
GraphDb db;
GraphDbAccessor dba(db);
auto vertex = dba.InsertVertex();
vertex.add_label(dba.Label("test"));
auto collection = dba.Vertices(dba.Label("test"), false);
std::vector<VertexAccessor> collection_vector(collection.begin(),
collection.end());
EXPECT_EQ(collection_vector.size(), (size_t)0);
}
// Test label index by adding and removing one vertex, and removing label from
// another, while the third one with an irrelevant label exists.
TEST(LabelsIndexDb, AddGetRemoveLabel) {
GraphDb db;
{
GraphDbAccessor dba(db);
auto vertex1 = dba.InsertVertex();
vertex1.add_label(dba.Label("test"));
auto vertex2 = dba.InsertVertex();
vertex2.add_label(dba.Label("test2"));
auto vertex3 = dba.InsertVertex();
vertex3.add_label(dba.Label("test"));
dba.Commit();
} // Finish transaction.
{
GraphDbAccessor dba(db);
auto filtered = dba.Vertices(dba.Label("test"), false);
std::vector<VertexAccessor> collection(filtered.begin(), filtered.end());
auto vertices = dba.Vertices(false);
std::vector<VertexAccessor> expected_collection;
for (auto vertex : vertices) {
if (vertex.has_label(dba.Label("test"))) {
expected_collection.push_back(vertex);
} else {
EXPECT_TRUE(vertex.has_label(dba.Label("test2")));
}
}
EXPECT_EQ(expected_collection.size(), collection.size());
EXPECT_TRUE(collection[0].has_label(dba.Label("test")));
EXPECT_TRUE(collection[1].has_label(dba.Label("test")));
EXPECT_FALSE(collection[0].has_label(dba.Label("test2")));
EXPECT_FALSE(collection[1].has_label(dba.Label("test2")));
dba.RemoveVertex(collection[0]); // Remove from database and test if
// index won't return it.
// Remove label from the vertex and add new label.
collection[1].remove_label(dba.Label("test"));
collection[1].add_label(dba.Label("test2"));
dba.Commit();
}
{
GraphDbAccessor dba(db);
auto filtered = dba.Vertices(dba.Label("test"), false);
std::vector<VertexAccessor> collection(filtered.begin(), filtered.end());
auto vertices = dba.Vertices(false);
std::vector<VertexAccessor> expected_collection;
for (auto vertex : vertices) {
if (vertex.has_label(dba.Label("test"))) {
expected_collection.push_back(vertex);
} else {
EXPECT_TRUE(vertex.has_label(dba.Label("test2")));
}
}
// It should be empty since everything with an old label is either deleted
// or doesn't have that label anymore.
EXPECT_EQ(expected_collection.size(), 0);
EXPECT_EQ(collection.size(), 0);
}
}
// TODO gleich - discuss with Flor the API changes and the tests
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}