tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement all edge filters for storage v2

Browse Source 
Summary:
Edge filters (edge type and destination vertex) are now handled natively in the
storage API. The API is implemented to be the fastest possible when using the
filters with the assumption that the number of edge types will be small.

Reviewers: teon.banek

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D2576
This commit is contained in:
Matej Ferencevic 2019-12-03 20:26:32 +01:00
parent f6b6ea254d
commit 10cc831ed2
9 changed files with 1261 additions and 732 deletions

42
src/query/db_accessor.hpp
View File

@ -123,9 +123,9 @@ class VertexAccessor final {
auto InEdges(storage::View view,
const std::vector<storage::EdgeType> &edge_types) const
-> storage::Result<decltype(
iter::imap(MakeEdgeAccessor, *impl_.InEdges(edge_types, view)))> {
auto maybe_edges = impl_.InEdges(edge_types, view);
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
@ -135,25 +135,18 @@ class VertexAccessor final {
auto InEdges(storage::View view,
const std::vector<storage::EdgeType> &edge_types,
const VertexAccessor &dest) const
-> storage::Result<decltype(
iter::imap(MakeEdgeAccessor, *impl_.InEdges(edge_types, view)))> {
auto maybe_edges = impl_.InEdges(edge_types, view);
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
std::vector<storage::EdgeAccessor> reduced_edges;
reduced_edges.reserve(maybe_edges->size());
for (auto &edge : *maybe_edges) {
if (edge.FromVertex() == dest.impl_) {
reduced_edges.push_back(edge);
}
}
return iter::imap(MakeEdgeAccessor, std::move(reduced_edges));
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::View view,
const std::vector<storage::EdgeType> &edge_types) const
-> storage::Result<decltype(
iter::imap(MakeEdgeAccessor, *impl_.OutEdges(edge_types, view)))> {
auto maybe_edges = impl_.OutEdges(edge_types, view);
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
@ -163,18 +156,11 @@ class VertexAccessor final {
auto OutEdges(storage::View view,
const std::vector<storage::EdgeType> &edge_types,
const VertexAccessor &dest) const
-> storage::Result<decltype(
iter::imap(MakeEdgeAccessor, *impl_.OutEdges(edge_types, view)))> {
auto maybe_edges = impl_.OutEdges(edge_types, view);
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
std::vector<storage::EdgeAccessor> reduced_edges;
reduced_edges.reserve(maybe_edges->size());
for (auto &edge : *maybe_edges) {
if (edge.ToVertex() == dest.impl_) {
reduced_edges.push_back(edge);
}
}
return iter::imap(MakeEdgeAccessor, std::move(reduced_edges));
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::Result<size_t> InDegree(storage::View view) const {

4
src/storage/v2/durability.cpp
View File

@ -1660,9 +1660,9 @@ void Durability::CreateSnapshot(Transaction *transaction) {
CHECK(maybe_labels.HasValue()) << "Invalid database state!";
auto maybe_props = va->Properties(View::OLD);
CHECK(maybe_props.HasValue()) << "Invalid database state!";
auto maybe_in_edges = va->InEdges({}, View::OLD);
auto maybe_in_edges = va->InEdges(View::OLD);
CHECK(maybe_in_edges.HasValue()) << "Invalid database state!";
auto maybe_out_edges = va->OutEdges({}, View::OLD);
auto maybe_out_edges = va->OutEdges(View::OLD);
CHECK(maybe_out_edges.HasValue()) << "Invalid database state!";
// Store the vertex.

86
src/storage/v2/vertex_accessor.cpp
View File

@ -348,7 +348,10 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(
}
Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(
const std::vector<EdgeTypeId> &edge_types, View view) const {
View view, const std::vector<EdgeTypeId> &edge_types,
const VertexAccessor *destination) const {
CHECK(!destination || destination->transaction_ == transaction_)
<< "Invalid accessor!";
bool exists = true;
bool deleted = false;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> in_edges;
@ -356,14 +359,33 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
in_edges = vertex_->in_edges;
if (edge_types.empty() && !destination) {
in_edges = vertex_->in_edges;
} else {
for (const auto &item : vertex_->in_edges) {
const auto &[edge_type, from_vertex, edge] = item;
if (destination && from_vertex != destination->vertex_) continue;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), edge_type) ==
edge_types.end())
continue;
in_edges.push_back(item);
}
}
delta = vertex_->delta;
}
ApplyDeltasForRead(
transaction_, delta, view,
[&exists, &deleted, &in_edges](const Delta &delta) {
[&exists, &deleted, &in_edges, &edge_types,
&destination](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_IN_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_)
break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(),
delta.vertex_edge.edge_type) == edge_types.end())
break;
// Add the edge because we don't see the removal.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{
delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
@ -374,6 +396,12 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(
break;
}
case Delta::Action::REMOVE_IN_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_)
break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(),
delta.vertex_edge.edge_type) == edge_types.end())
break;
// Remove the label because we don't see the addition.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{
delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
@ -405,18 +433,18 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::InEdges(
std::vector<EdgeAccessor> ret;
ret.reserve(in_edges.size());
for (const auto &item : in_edges) {
auto [edge_type, from_vertex, edge] = item;
if (edge_types.empty() || std::find(edge_types.begin(), edge_types.end(),
edge_type) != edge_types.end()) {
ret.emplace_back(edge, edge_type, from_vertex, vertex_, transaction_,
indices_, config_);
}
const auto &[edge_type, from_vertex, edge] = item;
ret.emplace_back(edge, edge_type, from_vertex, vertex_, transaction_,
indices_, config_);
}
return std::move(ret);
}
Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(
const std::vector<EdgeTypeId> &edge_types, View view) const {
View view, const std::vector<EdgeTypeId> &edge_types,
const VertexAccessor *destination) const {
CHECK(!destination || destination->transaction_ == transaction_)
<< "Invalid accessor!";
bool exists = true;
bool deleted = false;
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> out_edges;
@ -424,14 +452,33 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(
{
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
deleted = vertex_->deleted;
out_edges = vertex_->out_edges;
if (edge_types.empty() && !destination) {
out_edges = vertex_->out_edges;
} else {
for (const auto &item : vertex_->out_edges) {
const auto &[edge_type, to_vertex, edge] = item;
if (destination && to_vertex != destination->vertex_) continue;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(), edge_type) ==
edge_types.end())
continue;
out_edges.push_back(item);
}
}
delta = vertex_->delta;
}
ApplyDeltasForRead(
transaction_, delta, view,
[&exists, &deleted, &out_edges](const Delta &delta) {
[&exists, &deleted, &out_edges, &edge_types,
&destination](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_OUT_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_)
break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(),
delta.vertex_edge.edge_type) == edge_types.end())
break;
// Add the edge because we don't see the removal.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{
delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
@ -442,6 +489,12 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(
break;
}
case Delta::Action::REMOVE_OUT_EDGE: {
if (destination && delta.vertex_edge.vertex != destination->vertex_)
break;
if (!edge_types.empty() &&
std::find(edge_types.begin(), edge_types.end(),
delta.vertex_edge.edge_type) == edge_types.end())
break;
// Remove the label because we don't see the addition.
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{
delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
@ -473,12 +526,9 @@ Result<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(
std::vector<EdgeAccessor> ret;
ret.reserve(out_edges.size());
for (const auto &item : out_edges) {
auto [edge_type, to_vertex, edge] = item;
if (edge_types.empty() || std::find(edge_types.begin(), edge_types.end(),
edge_type) != edge_types.end()) {
ret.emplace_back(edge, edge_type, vertex_, to_vertex, transaction_,
indices_, config_);
}
const auto &[edge_type, to_vertex, edge] = item;
ret.emplace_back(edge, edge_type, vertex_, to_vertex, transaction_,
indices_, config_);
}
return std::move(ret);
}

7
src/storage/v2/vertex_accessor.hpp
View File

@ -65,18 +65,19 @@ class VertexAccessor final {
/// @throw std::bad_alloc
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
// TODO: Add API for obtaining edges filtered by destination VertexAccessor
/// @throw std::bad_alloc
/// @throw std::length_error if the resulting vector exceeds
/// std::vector::max_size().
Result<std::vector<EdgeAccessor>> InEdges(
const std::vector<EdgeTypeId> &edge_types, View view) const;
View view, const std::vector<EdgeTypeId> &edge_types = {},
const VertexAccessor *destination = nullptr) const;
/// @throw std::bad_alloc
/// @throw std::length_error if the resulting vector exceeds
/// std::vector::max_size().
Result<std::vector<EdgeAccessor>> OutEdges(
const std::vector<EdgeTypeId> &edge_types, View view) const;
View view, const std::vector<EdgeTypeId> &edge_types = {},
const VertexAccessor *destination = nullptr) const;
Result<size_t> InDegree(View view) const;

2
tests/unit/query_dump.cpp
View File

@ -135,7 +135,7 @@ DatabaseState GetState(storage::Storage *db) {
// Capture all edges
std::set<DatabaseState::Edge> edges;
for (const auto &vertex : dba.Vertices(storage::View::NEW)) {
auto maybe_edges = vertex.OutEdges({}, storage::View::NEW);
auto maybe_edges = vertex.OutEdges(storage::View::NEW);
CHECK(maybe_edges.HasValue());
for (const auto &edge : *maybe_edges) {
const auto &edge_type_name = dba.EdgeTypeToName(edge.EdgeType());

16
tests/unit/storage_v2.cpp
View File

@ -2275,9 +2275,9 @@ TEST(StorageV2, VertexNonexistentLabelPropertyEdgeAPI) {
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.GetProperty(property, storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.InEdges({}, storage::View::OLD).GetError(),
ASSERT_EQ(vertex.InEdges(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.OutEdges({}, storage::View::OLD).GetError(),
ASSERT_EQ(vertex.OutEdges(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.InDegree(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
@ -2290,8 +2290,8 @@ TEST(StorageV2, VertexNonexistentLabelPropertyEdgeAPI) {
ASSERT_EQ(vertex.Properties(storage::View::NEW)->size(), 0);
ASSERT_EQ(*vertex.GetProperty(property, storage::View::NEW),
storage::PropertyValue());
ASSERT_EQ(vertex.InEdges({}, storage::View::NEW)->size(), 0);
ASSERT_EQ(vertex.OutEdges({}, storage::View::NEW)->size(), 0);
ASSERT_EQ(vertex.InEdges(storage::View::NEW)->size(), 0);
ASSERT_EQ(vertex.OutEdges(storage::View::NEW)->size(), 0);
ASSERT_EQ(*vertex.InDegree(storage::View::NEW), 0);
ASSERT_EQ(*vertex.OutDegree(storage::View::NEW), 0);
@ -2311,9 +2311,9 @@ TEST(StorageV2, VertexNonexistentLabelPropertyEdgeAPI) {
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.GetProperty(property, storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.InEdges({}, storage::View::OLD).GetError(),
ASSERT_EQ(vertex.InEdges(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.OutEdges({}, storage::View::OLD).GetError(),
ASSERT_EQ(vertex.OutEdges(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
ASSERT_EQ(vertex.InDegree(storage::View::OLD).GetError(),
storage::Error::NONEXISTENT_OBJECT);
@ -2326,8 +2326,8 @@ TEST(StorageV2, VertexNonexistentLabelPropertyEdgeAPI) {
ASSERT_EQ(vertex.Properties(storage::View::NEW)->size(), 1);
ASSERT_EQ(*vertex.GetProperty(property, storage::View::NEW),
storage::PropertyValue("value"));
ASSERT_EQ(vertex.InEdges({}, storage::View::NEW)->size(), 1);
ASSERT_EQ(vertex.OutEdges({}, storage::View::NEW)->size(), 1);
ASSERT_EQ(vertex.InEdges(storage::View::NEW)->size(), 1);
ASSERT_EQ(vertex.OutEdges(storage::View::NEW)->size(), 1);
ASSERT_EQ(*vertex.InDegree(storage::View::NEW), 1);
ASSERT_EQ(*vertex.OutDegree(storage::View::NEW), 1);

24
tests/unit/storage_v2_durability.cpp
View File

@ -326,7 +326,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
acc.FindVertex(base_vertex_gids_[(i / 2) % kNumBaseVertices],
storage::View::OLD);
ASSERT_TRUE(vertex1);
auto out_edges = vertex1->OutEdges({}, storage::View::OLD);
auto out_edges = vertex1->OutEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
auto edge1 = find_edge(*out_edges);
ASSERT_TRUE(edge1);
@ -351,7 +351,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
acc.FindVertex(base_vertex_gids_[(i / 3) % kNumBaseVertices],
storage::View::OLD);
ASSERT_TRUE(vertex2);
auto in_edges = vertex2->InEdges({}, storage::View::OLD);
auto in_edges = vertex2->InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
auto edge2 = find_edge(*in_edges);
ASSERT_TRUE(edge2);
@ -479,7 +479,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
extended_vertex_gids_[(i / 5) % kNumExtendedVertices],
storage::View::OLD);
ASSERT_TRUE(vertex1);
auto out_edges = vertex1->OutEdges({}, storage::View::OLD);
auto out_edges = vertex1->OutEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
auto edge1 = find_edge(*out_edges);
ASSERT_TRUE(edge1);
@ -498,7 +498,7 @@ class DurabilityTest : public ::testing::TestWithParam<bool> {
extended_vertex_gids_[(i / 6) % kNumExtendedVertices],
storage::View::OLD);
ASSERT_TRUE(vertex2);
auto in_edges = vertex2->InEdges({}, storage::View::OLD);
auto in_edges = vertex2->InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
auto edge2 = find_edge(*in_edges);
ASSERT_TRUE(edge2);
@ -1141,7 +1141,7 @@ TEST_F(DurabilityTest,
{
auto acc = store.Access();
for (auto vertex : acc.Vertices(storage::View::OLD)) {
auto in_edges = vertex.InEdges({}, storage::View::OLD);
auto in_edges = vertex.InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
for (auto edge : *in_edges) {
// TODO (mferencevic): Replace with `ClearProperties()`
@ -1152,7 +1152,7 @@ TEST_F(DurabilityTest,
.HasValue());
}
}
auto out_edges = vertex.InEdges({}, storage::View::OLD);
auto out_edges = vertex.InEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
for (auto edge : *out_edges) {
// TODO (mferencevic): Replace with `ClearProperties()`
@ -1397,10 +1397,10 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) {
auto props = v1->Properties(storage::View::OLD);
ASSERT_TRUE(props.HasValue());
ASSERT_EQ(props->size(), 0);
auto in_edges = v1->InEdges({}, storage::View::OLD);
auto in_edges = v1->InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
ASSERT_EQ(in_edges->size(), 0);
auto out_edges = v1->OutEdges({}, storage::View::OLD);
auto out_edges = v1->OutEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
ASSERT_EQ(out_edges->size(), 1);
const auto &edge = (*out_edges)[0];
@ -1426,7 +1426,7 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) {
ASSERT_THAT(*props, UnorderedElementsAre(
std::make_pair(store.NameToProperty("hello"),
storage::PropertyValue("world"))));
auto in_edges = v2->InEdges({}, storage::View::OLD);
auto in_edges = v2->InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
ASSERT_EQ(in_edges->size(), 1);
const auto &edge = (*in_edges)[0];
@ -1440,7 +1440,7 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) {
} else {
ASSERT_EQ(edge_props->size(), 0);
}
auto out_edges = v2->OutEdges({}, storage::View::OLD);
auto out_edges = v2->OutEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
ASSERT_EQ(out_edges->size(), 0);
}
@ -1455,10 +1455,10 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) {
ASSERT_THAT(*props,
UnorderedElementsAre(std::make_pair(
store.NameToProperty("v3"), storage::PropertyValue(42))));
auto in_edges = v3->InEdges({}, storage::View::OLD);
auto in_edges = v3->InEdges(storage::View::OLD);
ASSERT_TRUE(in_edges.HasValue());
ASSERT_EQ(in_edges->size(), 0);
auto out_edges = v3->OutEdges({}, storage::View::OLD);
auto out_edges = v3->OutEdges(storage::View::OLD);
ASSERT_TRUE(out_edges.HasValue());
ASSERT_EQ(out_edges->size(), 0);
}

1808
tests/unit/storage_v2_edge.cpp
View File

File diff suppressed because it is too large Load Diff

4
tests/unit/storage_v2_gc.cpp
View File

@ -129,7 +129,7 @@ TEST(StorageV2Gc, Sanity) {
auto vertex = acc.FindVertex(vertices[i], storage::View::NEW);
EXPECT_EQ(vertex.has_value(), i % 5 != 0 && i % 3 != 0);
if (vertex.has_value()) {
auto out_edges = vertex->OutEdges({}, storage::View::NEW);
auto out_edges = vertex->OutEdges(storage::View::NEW);
if (i % 5 != 4 && i % 3 != 2) {
EXPECT_EQ(out_edges.GetValue().size(), 1);
EXPECT_EQ(*vertex->OutDegree(storage::View::NEW), 1);
@ -138,7 +138,7 @@ TEST(StorageV2Gc, Sanity) {
EXPECT_TRUE(out_edges->empty());
}
auto in_edges = vertex->InEdges({}, storage::View::NEW);
auto in_edges = vertex->InEdges(storage::View::NEW);
if (i % 5 != 1 && i % 3 != 1) {
EXPECT_EQ(in_edges.GetValue().size(), 1);
EXPECT_EQ(*vertex->InDegree(storage::View::NEW), 1);