tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extract address types

Browse Source 
Summary:
We have been using `Edges::VertexAddress` and `Edges::EdgeAddress` a lot
in other parts of the codebase because it's cleaner to write then
`Address<mvcc::VersionList<Edge>>`, especially in code what should not
really be MVCC-aware. However, a lot of that code should not really be
`Edges` aware either, as that's a storage datastructure that should not
be exposed.

This became annoying, so I extracted these addresses into a type-file. I
don't really like this approach, it might be better to have
`Vertex::Address` and `Edge::Address`, but that means we'd have to
import those headers and we'd get circular dependencies.

“The horror! The horror!”
   - Joseph Conrad, Heart of Darkness

Reviewers: teon.banek, buda

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1204
This commit is contained in:
florijan 2018-02-15 15:03:30 +01:00
parent f83cc31779
commit b2d7f95568
13 changed files with 81 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/database/graph_db_accessor.cpp
View File

@ -6,6 +6,7 @@
#include "database/graph_db_accessor.hpp"
#include "database/state_delta.hpp"
#include "distributed/index_rpc_messages.hpp"
#include "storage/address_types.hpp"
#include "storage/edge.hpp"
#include "storage/edge_accessor.hpp"
#include "storage/vertex.hpp"
@ -415,8 +416,8 @@ EdgeAccessor GraphDbAccessor::InsertEdge(
edge_type);
}
EdgeAccessor GraphDbAccessor::InsertOnlyEdge(Edges::VertexAddress &from,
Edges::VertexAddress &to,
EdgeAccessor GraphDbAccessor::InsertOnlyEdge(storage::VertexAddress &from,
storage::VertexAddress &to,
storage::EdgeType edge_type,
gid::Gid edge_gid) {
auto gid = db_.storage().edge_generator_.Next(edge_gid);

5
src/database/graph_db_accessor.hpp
View File

@ -9,6 +9,7 @@
#include "database/graph_db.hpp"
#include "distributed/remote_cache.hpp"
#include "storage/address_types.hpp"
#include "storage/edge_accessor.hpp"
#include "storage/types.hpp"
#include "storage/vertex_accessor.hpp"
@ -280,8 +281,8 @@ class GraphDbAccessor {
* Insert edge into main storage, but don't insert it into from and to
* vertices edge lists.
*/
EdgeAccessor InsertOnlyEdge(Edges::VertexAddress &from,
Edges::VertexAddress &to,
EdgeAccessor InsertOnlyEdge(storage::VertexAddress &from,
storage::VertexAddress &to,
storage::EdgeType edge_type, gid::Gid edge_gid);
/**

24
src/distributed/remote_pull_produce_rpc_messages.hpp
View File

@ -11,7 +11,7 @@
#include "distributed/serialization.hpp"
#include "query/frontend/semantic/symbol.hpp"
#include "query/parameters.hpp"
#include "storage/edges.hpp"
#include "storage/address_types.hpp"
#include "transactions/type.hpp"
#include "utils/serialization.hpp"
@ -258,16 +258,16 @@ struct RemotePullResData {
switch (type) {
case query::TypedValue::Type::Vertex: {
Edges::VertexAddress::StorageT address;
storage::VertexAddress::StorageT address;
ar >> address;
vertices.emplace_back(Edges::VertexAddress(address), load_vertex(ar),
vertices.emplace_back(storage::VertexAddress(address), load_vertex(ar),
load_vertex(ar), &value);
break;
}
case query::TypedValue::Type::Edge: {
Edges::VertexAddress::StorageT address;
storage::VertexAddress::StorageT address;
ar >> address;
edges.emplace_back(Edges::EdgeAddress(address), load_edge(ar),
edges.emplace_back(storage::EdgeAddress(address), load_edge(ar),
load_edge(ar), &value);
break;
}
@ -278,20 +278,20 @@ struct RemotePullResData {
paths.emplace_back(value);
auto &path_data = paths.back();
Edges::VertexAddress::StorageT vertex_address;
Edges::EdgeAddress::StorageT edge_address;
storage::VertexAddress::StorageT vertex_address;
storage::EdgeAddress::StorageT edge_address;
ar >> vertex_address;
path_data.vertices.emplace_back(Edges::VertexAddress(vertex_address),
path_data.vertices.emplace_back(storage::VertexAddress(vertex_address),
load_vertex(ar), load_vertex(ar),
nullptr);
for (size_t i = 0; i < path_size; ++i) {
ar >> edge_address;
path_data.edges.emplace_back(Edges::EdgeAddress(edge_address),
path_data.edges.emplace_back(storage::EdgeAddress(edge_address),
load_edge(ar), load_edge(ar), nullptr);
ar >> vertex_address;
path_data.vertices.emplace_back(Edges::VertexAddress(vertex_address),
load_vertex(ar), load_vertex(ar),
nullptr);
path_data.vertices.emplace_back(
storage::VertexAddress(vertex_address), load_vertex(ar),
load_vertex(ar), nullptr);
}
break;
}

3
src/distributed/serialization.hpp
View File

@ -4,6 +4,7 @@
#include <memory>
#include <vector>
#include "storage/address_types.hpp"
#include "storage/edge.hpp"
#include "storage/types.hpp"
#include "storage/vertex.hpp"
@ -99,7 +100,7 @@ void SaveElement(TArchive &ar, const Vertex &record, int worker_id) {
namespace impl {
template <typename TArchive>
Edges::VertexAddress LoadVertexAddress(TArchive &ar) {
storage::VertexAddress LoadVertexAddress(TArchive &ar) {
gid::Gid vertex_id;
ar >> vertex_id;
int worker_id;

29
src/durability/recovery.cpp
View File

@ -12,6 +12,7 @@
#include "durability/version.hpp"
#include "durability/wal.hpp"
#include "query/typed_value.hpp"
#include "storage/address_types.hpp"
#include "transactions/type.hpp"
#include "utils/algorithm.hpp"
@ -114,7 +115,7 @@ bool RecoverSnapshot(const fs::path &snapshot_file, database::GraphDb &db,
database::GraphDbAccessor dba(db);
std::unordered_map<gid::Gid,
std::pair<Edges::VertexAddress, Edges::VertexAddress>>
std::pair<storage::VertexAddress, storage::VertexAddress>>
edge_gid_endpoints_mapping;
for (int64_t i = 0; i < vertex_count; ++i) {
auto vertex = decoder.ReadSnapshotVertex();
@ -142,22 +143,22 @@ bool RecoverSnapshot(const fs::path &snapshot_file, database::GraphDb &db,
}
}
auto vertex_transform_to_local_if_possible =
[&db, &dba](Edges::VertexAddress &address) {
if (address.is_local()) return;
// If the worker id matches it should be a local apperance
if (address.worker_id() == db.WorkerId()) {
address = Edges::VertexAddress(dba.LocalVertexAddress(address.gid()));
CHECK(address.is_local()) << "Address should be local but isn't";
}
};
auto vertex_transform_to_local_if_possible = [&db, &dba](
storage::VertexAddress &address) {
if (address.is_local()) return;
// If the worker id matches it should be a local apperance
if (address.worker_id() == db.WorkerId()) {
address = storage::VertexAddress(dba.LocalVertexAddress(address.gid()));
CHECK(address.is_local()) << "Address should be local but isn't";
}
};
auto edge_transform_to_local_if_possible =
[&db, &dba](Edges::EdgeAddress &address) {
[&db, &dba](storage::EdgeAddress &address) {
if (address.is_local()) return;
// If the worker id matches it should be a local apperance
if (address.worker_id() == db.WorkerId()) {
address = Edges::EdgeAddress(dba.LocalEdgeAddress(address.gid()));
address = storage::EdgeAddress(dba.LocalEdgeAddress(address.gid()));
CHECK(address.is_local()) << "Address should be local but isn't";
}
};
@ -171,8 +172,8 @@ bool RecoverSnapshot(const fs::path &snapshot_file, database::GraphDb &db,
// edges must be bolt-compliant
auto &edge_endpoints = edge_gid_endpoints_mapping[edge.id];
Edges::VertexAddress from;
Edges::VertexAddress to;
storage::VertexAddress from;
storage::VertexAddress to;
std::tie(from, to) = edge_endpoints;
// From and to are written in the global_address format and we should

5
src/durability/snapshot_decoded_value.hpp
View File

@ -6,6 +6,7 @@
#include "communication/bolt/v1/decoder/decoded_value.hpp"
#include "query/typed_value.hpp"
#include "storage/address_types.hpp"
#include "storage/property_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/exceptions.hpp"
@ -35,8 +36,8 @@ struct DecodedSnapshotVertex {
struct DecodedInlinedVertexEdge {
// Addresses down below must always be global_address and never direct
// pointers to a record.
Edges::EdgeAddress address;
Edges::VertexAddress vertex;
storage::EdgeAddress address;
storage::VertexAddress vertex;
std::string type;
};

12
src/storage/address_types.hpp Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#include "mvcc/version_list.hpp"
#include "storage/address.hpp"
class Edge;
class Vertex;
namespace storage {
using VertexAddress = Address<mvcc::VersionList<Vertex>>;
using EdgeAddress = Address<mvcc::VersionList<Edge>>;
} // namespace storage

5
src/storage/edge_accessor.hpp
View File

@ -1,5 +1,6 @@
#pragma once
#include "storage/address_types.hpp"
#include "storage/edge.hpp"
#include "storage/record_accessor.hpp"
@ -19,8 +20,8 @@ class VertexAccessor;
* location, which is often a performance bottleneck in traversals.
*/
class EdgeAccessor : public RecordAccessor<Edge> {
using VertexAddress = storage::Address<mvcc::VersionList<Vertex>>;
using EdgeAddress = storage::Address<mvcc::VersionList<Edge>>;
using EdgeAddress = storage::EdgeAddress;
using VertexAddress = storage::VertexAddress;
public:
/** Constructor that reads data from the random memory location (lower

31
src/storage/edges.hpp
View File

@ -7,27 +7,20 @@
#include "mvcc/version_list.hpp"
#include "storage/address.hpp"
#include "storage/address_types.hpp"
#include "storage/types.hpp"
#include "utils/algorithm.hpp"
// forward declare Vertex and Edge because they need this data structure
class Edge;
class Vertex;
/**
* A data stucture that holds a number of edges. This implementation assumes
* that separate Edges instances are used for incoming and outgoing edges in a
* vertex (and consequently that edge Addresses are unique in it).
*/
class Edges {
public:
using VertexAddress = storage::Address<mvcc::VersionList<Vertex>>;
using EdgeAddress = storage::Address<mvcc::VersionList<Edge>>;
private:
struct Element {
VertexAddress vertex;
EdgeAddress edge;
storage::VertexAddress vertex;
storage::EdgeAddress edge;
storage::EdgeType edge_type;
};
@ -55,7 +48,8 @@ class Edges {
* If nullptr edges are not filtered on type.
*/
Iterator(std::vector<Element>::const_iterator position,
std::vector<Element>::const_iterator end, VertexAddress vertex,
std::vector<Element>::const_iterator end,
storage::VertexAddress vertex,
const std::vector<storage::EdgeType> *edge_types)
: position_(position),
end_(end),
@ -86,7 +80,7 @@ class Edges {
// Optional predicates. If set they define which edges are skipped by the
// iterator. Only one can be not-null in the current implementation.
VertexAddress vertex_{nullptr};
storage::VertexAddress vertex_{nullptr};
// For edge types we use a vector pointer because it's optional.
const std::vector<storage::EdgeType> *edge_types_ = nullptr;
@ -94,10 +88,9 @@ class Edges {
* present in this iterator. */
void update_position() {
if (vertex_.local()) {
position_ = std::find_if(position_,
end_, [v = this->vertex_](const Element &e) {
return e.vertex == v;
});
position_ = std::find_if(
position_, end_,
[v = this->vertex_](const Element &e) { return e.vertex == v; });
}
if (edge_types_) {
position_ = std::find_if(position_, end_, [this](const Element &e) {
@ -116,7 +109,7 @@ class Edges {
* @param edge - The edge.
* @param edge_type - Type of the edge.
*/
void emplace(VertexAddress vertex, EdgeAddress edge,
void emplace(storage::VertexAddress vertex, storage::EdgeAddress edge,
storage::EdgeType edge_type) {
storage_.emplace_back(Element{vertex, edge, edge_type});
}
@ -124,7 +117,7 @@ class Edges {
/**
* Removes an edge from this structure.
*/
void RemoveEdge(EdgeAddress edge) {
void RemoveEdge(storage::EdgeAddress edge) {
auto found = std::find_if(
storage_.begin(), storage_.end(),
[edge](const Element &element) { return edge == element.edge; });
@ -146,7 +139,7 @@ class Edges {
* @param edge_types - The edge types at least one of which must be matched.
* If nullptr edges are not filtered on type.
*/
auto begin(VertexAddress vertex,
auto begin(storage::VertexAddress vertex,
const std::vector<storage::EdgeType> *edge_types) const {
if (edge_types && edge_types->empty()) edge_types = nullptr;
return Iterator(storage_.begin(), storage_.end(), vertex, edge_types);

4
src/storage/record_accessor.cpp
View File

@ -97,13 +97,13 @@ gid::Gid RecordAccessor<TRecord>::gid() const {
}
template <typename TRecord>
storage::Address<mvcc::VersionList<TRecord>> RecordAccessor<TRecord>::address()
typename RecordAccessor<TRecord>::AddressT RecordAccessor<TRecord>::address()
const {
return address_;
}
template <typename TRecord>
storage::Address<mvcc::VersionList<TRecord>>
typename RecordAccessor<TRecord>::AddressT
RecordAccessor<TRecord>::GlobalAddress() const {
return is_local() ? storage::Address<mvcc::VersionList<TRecord>>(
gid(), db_accessor_->db_.WorkerId())

9
tests/manual/card_fraud_generate_snapshot.cpp
View File

@ -12,6 +12,7 @@
#include "durability/paths.hpp"
#include "durability/snapshot_encoder.hpp"
#include "durability/version.hpp"
#include "storage/address_types.hpp"
#include "utils/datetime/timestamp.hpp"
#include "utils/string.hpp"
#include "utils/timer.hpp"
@ -226,9 +227,9 @@ class Writer {
encoder_.WriteInt(in_edges.size());
for (auto &edge : in_edges) {
auto edge_addr = Edges::EdgeAddress(edge.gid, edge.from.first);
auto edge_addr = storage::EdgeAddress(edge.gid, edge.from.first);
auto vertex_addr =
Edges::VertexAddress(edge.from.second, edge.from.first);
storage::VertexAddress(edge.from.second, edge.from.first);
encoder_.WriteInt(edge_addr.raw());
encoder_.WriteInt(vertex_addr.raw());
encoder_.WriteString(
@ -237,8 +238,8 @@ class Writer {
encoder_.WriteInt(out_edges.size());
for (auto &edge : out_edges) {
auto edge_addr = Edges::EdgeAddress(edge.gid, edge.from.first);
auto vertex_addr = Edges::VertexAddress(edge.to.second, edge.to.first);
auto edge_addr = storage::EdgeAddress(edge.gid, edge.from.first);
auto vertex_addr = storage::VertexAddress(edge.to.second, edge.to.first);
encoder_.WriteInt(edge_addr.raw());
encoder_.WriteInt(vertex_addr.raw());
encoder_.WriteString(

3
tests/unit/distributed_common.hpp
View File

@ -4,6 +4,7 @@
#include "database/graph_db.hpp"
#include "database/graph_db_accessor.hpp"
#include "storage/address_types.hpp"
#include "transactions/engine_master.hpp"
class DistributedGraphDbTest : public ::testing::Test {
@ -74,7 +75,7 @@ class DistributedGraphDbTest : public ::testing::Test {
}
/// Inserts an edge (on the 'from' side) and returns it's global address.
auto InsertEdge(Edges::VertexAddress from, Edges::VertexAddress to,
auto InsertEdge(storage::VertexAddress from, storage::VertexAddress to,
const std::string &edge_type_name) {
database::GraphDbAccessor dba{worker(from.worker_id())};
auto from_v = dba.FindVertexChecked(from.gid(), false);

13
tools/src/mg_import_csv/main.cpp
View File

@ -16,6 +16,7 @@
#include "durability/snapshot_decoded_value.hpp"
#include "durability/snapshot_encoder.hpp"
#include "durability/version.hpp"
#include "storage/address_types.hpp"
#include "utils/string.hpp"
#include "utils/timer.hpp"
@ -378,12 +379,12 @@ void Convert(const std::vector<std::string> &nodes,
}
for (auto edge : edges) {
auto encoded = edge.second;
vertices[encoded.from].out.push_back({Edges::EdgeAddress(encoded.id, 0),
Edges::VertexAddress(encoded.to, 0),
encoded.type});
vertices[encoded.to].in.push_back({Edges::EdgeAddress(encoded.id, 0),
Edges::VertexAddress(encoded.from, 0),
encoded.type});
vertices[encoded.from].out.push_back(
{storage::EdgeAddress(encoded.id, 0),
storage::VertexAddress(encoded.to, 0), encoded.type});
vertices[encoded.to].in.push_back(
{storage::EdgeAddress(encoded.id, 0),
storage::VertexAddress(encoded.from, 0), encoded.type});
}
for (auto vertex_pair : vertices) {
auto &vertex = vertex_pair.second;