Clean docs folder
Summary: Removed folders: * dev/diagram * dev/distributed * feature_ref Reviewers: teon.banek, mferencevic Reviewed By: mferencevic Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D2710
This commit is contained in:
Marko Budiselic
parent
000d6dba55
commit
32b35f6f88
@ -1,288 +0,0 @@
|
||||
// dot -Tpng dependencies.dot -o /path/to/output.png
|
||||
|
||||
// TODO (buda): Put PropertyValueStore to storage namespace
|
||||
|
||||
digraph {
|
||||
// At the beginning of each block there is a default style for that block
|
||||
label="Memgraph Dependencies Diagram"; fontname="Roboto Bold"; fontcolor=black;
|
||||
fontsize=26; labelloc=top; labeljust=right;
|
||||
compound=true; // If true, allow edges between clusters
|
||||
rankdir=TB; // Alternatives: LR
|
||||
node [shape=record fontname="Roboto", fontsize=12, fontcolor=white];
|
||||
edge [color="#B5AFB7"];
|
||||
|
||||
// -- Legend --
|
||||
// dir=both arrowtail=diamond arrowhead=vee -> group ownership
|
||||
// dir=both arrowtail=none, arrowhead=vee -> ownership; stack or uptr
|
||||
|
||||
subgraph cluster_tcp_end_client_communication {
|
||||
label="TCP End Client Communication"; fontsize=14;
|
||||
node [style=filled, color="#DD2222" fillcolor="#DD2222"];
|
||||
|
||||
// Owned elements
|
||||
"communication::Server";
|
||||
"io::network::Socket";
|
||||
|
||||
// Intracluster connections
|
||||
"communication::Server" -> "io::network::Socket"
|
||||
[label="socket_" dir=both arrowtail=none arrowhead=vee];
|
||||
}
|
||||
|
||||
subgraph cluster_bolt_server {
|
||||
label="Bolt Server"; fontsize=14;
|
||||
node [style=filled, color="#62A2CA" fillcolor="#62A2CA"];
|
||||
|
||||
// Owned elements
|
||||
"communication::bolt::SessionData";
|
||||
"communication::bolt::Session";
|
||||
"communication::bolt::Encoder";
|
||||
"communication::bolt::Decoder";
|
||||
|
||||
// Intracluster connections
|
||||
"communication::bolt::Session" -> "communication::bolt::Encoder"
|
||||
[label="encoder_", dir=both arrowtail=none, arrowhead=vee];
|
||||
"communication::bolt::Session" -> "communication::bolt::Decoder"
|
||||
[label="decoder_", dir=both arrowtail=none, arrowhead=vee];
|
||||
}
|
||||
|
||||
subgraph cluster_opencypher_engine {
|
||||
label="openCypher Engine"; fontsize=14;
|
||||
node [style=filled, color="#68BDF6" fillcolor="#68BDF6"];
|
||||
|
||||
// Owned Elements
|
||||
"query::Interpreter";
|
||||
"query::AstTreeStorage";
|
||||
"query::TypedValue"
|
||||
"query::Path";
|
||||
"query::Simbol";
|
||||
"query::Context";
|
||||
"query::ExpressionEvaluator";
|
||||
"query::Frame";
|
||||
"query::SymbolTable";
|
||||
"query::plan::LogicalOperator";
|
||||
"query::plan::Cursor";
|
||||
"query::plan::CostEstimator";
|
||||
|
||||
// Intracluster connections
|
||||
"query::Interpreter" -> "query::AstTreeStorage"
|
||||
[label="ast_cache" dir=both arrowtail=diamond arrowhead=vee];
|
||||
"query::TypedValue" -> "query::Path";
|
||||
"query::plan::Cursor" -> "query::Frame";
|
||||
"query::plan::Cursor" -> "query::Context";
|
||||
"query::plan::LogicalOperator" -> "query::Symbol";
|
||||
"query::plan::LogicalOperator" -> "query::SymbolTable";
|
||||
"query::plan::LogicalOperator" -> "query::plan::Cursor";
|
||||
}
|
||||
|
||||
|
||||
subgraph cluster_storage {
|
||||
label="Storage" fontsize=14;
|
||||
node [style=filled, color="#FB6E00" fillcolor="#FB6E00"];
|
||||
|
||||
// Owned Elements
|
||||
"database::GraphDb";
|
||||
"database::GraphDbAccessor";
|
||||
"storage::Record";
|
||||
"storage::Vertex";
|
||||
"storage::Edge";
|
||||
"storage::RecordAccessor";
|
||||
"storage::VertexAccessor";
|
||||
"storage::EdgeAccessor";
|
||||
"storage::Common";
|
||||
"storage::Label";
|
||||
"storage::EdgeType";
|
||||
"storage::Property";
|
||||
"storage::compression";
|
||||
"storage::SingleNodeConcurrentIdMapper";
|
||||
"storage::Location";
|
||||
"storage::StorageTypesLocation";
|
||||
"PropertyValueStore";
|
||||
"storage::RecordLock";
|
||||
"mvcc::Version";
|
||||
"mvcc::Record";
|
||||
"mvcc::VersionList";
|
||||
|
||||
// Intracluster connections
|
||||
"storage::VertexAccessor" -> "storage::RecordAccessor"
|
||||
[arrowhead=onormal];
|
||||
"storage::EdgeAccessor" -> "storage::RecordAccessor"
|
||||
[arrowhead=onormal];
|
||||
"storage::RecordAccessor" -> "database::GraphDbAccessor"
|
||||
[style=dashed arrowhead=vee];
|
||||
"storage::Vertex" -> "mvcc::Record"
|
||||
[arrowhead=onormal];
|
||||
"storage::Edge" -> "mvcc::Record"
|
||||
[arrowhead=onormal];
|
||||
"storage::Edge" -> "PropertyValueStore"
|
||||
[arrowhead=vee];
|
||||
"storage::Vertex" -> "PropertyValueStore"
|
||||
[arrowhead=vee];
|
||||
"storage::Edge" -> "mvcc::VersionList"
|
||||
[label="from,to" arrowhead=vee style=dashed];
|
||||
"storage::VertexAccessor" -> "storage::Vertex"
|
||||
[arrowhead=vee];
|
||||
"storage::EdgeAccessor" -> "storage::Edge"
|
||||
[arrowhead=vee];
|
||||
"storage::SingleNodeConcurrentIdMapper" -> "storage::StorageTypesLocation"
|
||||
[arrowhead=vee];
|
||||
"storage::StorageTypesLocation" -> "storage::Location"
|
||||
[arrowhead=vee];
|
||||
"storage::Storage" -> "storage::StorageTypesLocation"
|
||||
[arrowhead=vee];
|
||||
"storage::Property" -> "storage::Common"
|
||||
[arrowhead=onormal];
|
||||
"storage::Label" -> "storage::Common"
|
||||
[arrowhead=onormal];
|
||||
"storage::EdgeType" -> "storage::Common"
|
||||
[arrowhead=onormal];
|
||||
"storage::Property" -> "storage::Location"
|
||||
[arrowhead=vee];
|
||||
"PropertyValueStore" -> "storage::Property"
|
||||
[arrowhead=vee];
|
||||
"PropertyValueStore" -> "storage::Location"
|
||||
[arrowhead=vee];
|
||||
"database::GraphDbAccessor" -> "database::GraphDb"
|
||||
[arrowhead=vee];
|
||||
"database::GraphDbAccessor" -> "tx::TransactionId"
|
||||
[arrowhead=vee];
|
||||
"mvcc::VersionList" -> "storge::RecordLock"
|
||||
[label="lock" arrowhead=vee];
|
||||
"mvcc::VersionList" -> "mvcc::Record"
|
||||
[label="head" arrowhead=vee];
|
||||
"mvcc::Record" -> "mvcc::Version"
|
||||
[arrowhead=onormal];
|
||||
|
||||
// Explicit positioning
|
||||
{rank=same;
|
||||
"database::GraphDbAccessor";
|
||||
"storage::VertexAccessor";
|
||||
"storage::EdgeAccessor";}
|
||||
{rank=same;
|
||||
"storage::Common";
|
||||
"storage::compression";}
|
||||
}
|
||||
|
||||
subgraph cluster_properties_on_disk {
|
||||
label="Properties on Disk" fontsize=14;
|
||||
node [style=filled, color="#102647" fillcolor="#102647"];
|
||||
|
||||
// Owned Elements
|
||||
"storage::KVStore";
|
||||
"rocksdb";
|
||||
|
||||
// Intracluster connections
|
||||
"storage::KVStore" -> "rocksdb";
|
||||
}
|
||||
|
||||
subgraph cluster_distributed {
|
||||
label="Distributed" fontsize=14;
|
||||
node [style=filled, color="#FFC500" fillcolor="#FFC500"];
|
||||
|
||||
// Owned Elements
|
||||
"distributed::DataManager";
|
||||
"distributed::DataRpcClients";
|
||||
|
||||
// Intracluster connections
|
||||
"distributed::DataManager" -> "distributed::DataRpcClients"
|
||||
[arrowhead=vee];
|
||||
"storage::RecordAccessor" -> "distributed::DataManager"
|
||||
[style=dashed arrowhead=vee];
|
||||
}
|
||||
|
||||
subgraph cluster_dynamic_partitioning {
|
||||
label="Dynamic Partitioning" fontsize=14;
|
||||
node [style=filled, color="#720096" fillcolor="#720096"];
|
||||
|
||||
// Owned Elements
|
||||
"DynamicPartitioner";
|
||||
}
|
||||
|
||||
subgraph cluster_security {
|
||||
label="Security" fontsize=14;
|
||||
node [style=filled, color="#857F87" fillcolor="#857F87"];
|
||||
|
||||
// Owned Elements
|
||||
"Communication Encryption";
|
||||
"Data Encryption";
|
||||
"Access Control";
|
||||
"Audit Logging";
|
||||
}
|
||||
|
||||
subgraph cluster_web_dashboard {
|
||||
label="Dashaboard" fontsize=14;
|
||||
node [style=filled, color="#FF0092" fillcolor="#FF0092"];
|
||||
|
||||
// Owned Elements
|
||||
"Memgraph Ops / Memgraph Cockpit";
|
||||
}
|
||||
|
||||
subgraph cluster_rpc {
|
||||
label="RPC" fontsize=14;
|
||||
node [style=filled, color="#857F87" fillcolor="#857F87"];
|
||||
|
||||
// Owned Elements
|
||||
"communication::rpc::Server";
|
||||
"communication::rpc::Client";
|
||||
}
|
||||
|
||||
subgraph cluster_ingestion {
|
||||
label="Ingestion" fontsize=14;
|
||||
node [style=filled, color="#0B6D88" fillcolor="#0B6D88"];
|
||||
|
||||
// Owned Elements
|
||||
"Extract";
|
||||
"Transform";
|
||||
"Load";
|
||||
"Amazon S3";
|
||||
"Kafka";
|
||||
|
||||
// Intracluster connections
|
||||
"Extract" -> "Amazon S3";
|
||||
"Extract" -> "Kafka";
|
||||
|
||||
// Explicit positioning
|
||||
{rank=same;"Extract";"Transform";"Load";}
|
||||
}
|
||||
|
||||
// -- Intercluster connections --
|
||||
// cluster_tcp_end_client_communication -- cluster_bolt_server
|
||||
"communication::Server" -> "communication::bolt::SessionData" [color=black];
|
||||
"communication::Server" -> "communication::bolt::Session" [color=black];
|
||||
// cluster_bolt_server -> cluster_storage
|
||||
"communication::bolt::SessionData" -> "database::GraphDb" [color=red];
|
||||
"communication::bolt::Session" -> "database::GraphDbAccessor" [color=red];
|
||||
// cluster_bolt_server -> cluster_opencypher_engine
|
||||
"communication::bolt::SessionData" -> "query::Interpreter" [color=red];
|
||||
// cluster_opencypher_engine -- cluster_storage
|
||||
"query::Interpreter" -> "database::GraphDbAccessor" [color=black];
|
||||
"query::Interpreter" -> "storage::VertexAccessor" [color=black];
|
||||
"query::Interpreter" -> "storage::EdgeAccessor" [color=black];
|
||||
"query::TypedValue" -> "storage::VertexAccessor" [color=black];
|
||||
"query::TypedValue" -> "storage::EdgeAccessor" [color=black];
|
||||
"query::Path" -> "storage::VertexAccessor"
|
||||
[label="vertices" dir=both arrowtail=diamond arrowhead=vee color=black];
|
||||
"query::Path" -> "storage::EdgeAccessor"
|
||||
[label="edges" dir=both arrowtail=diamond arrowhead=vee color=black];
|
||||
"query::plan::LogicalOperator" -> "database::GraphDbAccessor"
|
||||
[color=black arrowhead=vee];
|
||||
// cluster_distributed -- cluster_storage
|
||||
"distributed::DataManager" -> "database::GraphDb"
|
||||
[arrowhead=vee style=dashed color=red];
|
||||
"distributed::DataManager" -> "tx::TransactionId"
|
||||
[label="ves_caches_key" dir=both arrowhead=none arrowtail=diamond
|
||||
color=red];
|
||||
"distributed::DataManager" -> "storage::Vertex"
|
||||
[label="vertices_caches" dir=both arrowhead=none arrowtail=diamond
|
||||
color=red];
|
||||
"distributed::DataManager" -> "storage::Edge"
|
||||
[label="edges_caches" dir=both arrowhead=none arrowtail=diamond
|
||||
color=red];
|
||||
// cluster_storage -- cluster_properties_on_disk
|
||||
"PropertyValueStore" -> "storage::KVStore"
|
||||
[label="static" arrowhead=vee color=black];
|
||||
// cluster_dynamic_partitioning -- cluster_storage
|
||||
"database::GraphDb" -> "DynamicPartitioner"
|
||||
[arrowhead=vee color=red];
|
||||
"DynamicPartitioner" -> "database::GraphDbAccessor"
|
||||
[arrowhead=vee color=black];
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
digraph {
|
||||
// label="Dynamig Graph Partitioning";
|
||||
fontname="Roboto Bold"; fontcolor=black;
|
||||
fontsize=26; labelloc=top; labeljust=center;
|
||||
compound=true; // If true, allow edges between clusters
|
||||
rankdir=TB; // Alternatives: LR
|
||||
node [shape=record fontname="Roboto", fontsize=12, fontcolor=white
|
||||
style=filled, color="#FB6E00" fillcolor="#FB6E00"];
|
||||
edge [color="#B5AFB7"];
|
||||
|
||||
"distributed::DistributedGraphDb" -> "distributed::TokenSharingRpcServer";
|
||||
|
||||
"distributed::TokenSharingRpcServer" -> "communication::rpc::Server";
|
||||
"distributed::TokenSharingRpcServer" -> "distributed::Coordination";
|
||||
"distributed::TokenSharingRpcServer" -> "distributed::TokenSharingRpcClients";
|
||||
"distributed::TokenSharingRpcServer" -> "distributed::dgp::Partitioner";
|
||||
|
||||
"distributed::dgp::Partitioner" -> "distributed::DistributedGraphDb" [style=dashed];
|
||||
|
||||
"distributed::dgp::Partitioner" -> "distributed::dgp::VertexMigrator";
|
||||
"distributed::dgp::VertexMigrator" -> "database::GraphDbAccessor" [style=dashed];
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
# Distributed addressing
|
||||
|
||||
In distributed Memgraph a single graph element must be owned by exactly
|
||||
one worker. It is possible that multiple workers have cached copies of
|
||||
a single graph element (which is inevitable), but there is only one
|
||||
owner.
|
||||
|
||||
The owner of a graph element can change. This is not yet implemented,
|
||||
but is intended. Graph partitioning is intended to be dynamic.
|
||||
|
||||
Graph elements refer to other graph elements that are possibly on some
|
||||
other worker. Even though each graph element is identified with a unique
|
||||
ID, that ID does not contain the information about where that element
|
||||
currently resides (which worker is the owner).
|
||||
|
||||
Thus we introduce the concept of a global address. It indicates both
|
||||
which graph element is referred to (it's global ID), and where it
|
||||
resides. Semantically it's a pair of two elements, but for efficiency
|
||||
it's stored in 64 bits.
|
||||
|
||||
The global address is efficient for usage in a cluster: it indicates
|
||||
where something can be found. However, finding a graph element based on
|
||||
it's ID is still not a free operation (in the current implementation
|
||||
it's a skiplist lookup). So, whenever possible, it's better to use local
|
||||
addresses (pointers).
|
||||
|
||||
Succinctly, the requirements for addressing are:
|
||||
- global addressing containing location info
|
||||
- fast local addressing
|
||||
- storage of both types in the same location efficiently
|
||||
- translation between the two
|
||||
|
||||
The `storage::Address` class handles the enumerated storage
|
||||
requirements. It stores either a local or global address in the size of
|
||||
a local pointer (typically 8 bytes).
|
||||
|
||||
Conversion between the two is done in multiple places. The general
|
||||
approach is to use local addresses (when possible) only for local
|
||||
in-memory handling. All the communication and persistence uses global
|
||||
addresses. Also, when receiving address from another worker, attempt to
|
||||
localize addresses as soon as possible, so that least code has to worry
|
||||
about potential inefficiency of using a global address for a local graph
|
||||
element.
|
||||
@ -1,50 +0,0 @@
|
||||
# Distributed durability
|
||||
|
||||
Durability in distributed is slightly different then in single-node as
|
||||
the state itself is shared between multiple workers and none of those
|
||||
states are independent.
|
||||
|
||||
Note that recovering from persistent storage must result in a stable
|
||||
database state. This means that across the cluster the state
|
||||
modification of every transaction that was running is either recovered
|
||||
fully or not at all. Also, if transaction A committed before transaction B,
|
||||
then if B is recovered so must A.
|
||||
|
||||
## Snapshots
|
||||
|
||||
It is possibly avoidable but highly desirable that the database can be
|
||||
recovered from snapshot only, without relying on WAL files. For this to
|
||||
be possible in distributed, it must be ensured that the same
|
||||
transactions are recovered on all the workers (including master) in the
|
||||
cluster. Since the snapshot does not contain information about which
|
||||
state change happened in which transaction, the only way to achieve this
|
||||
is to have synchronized snapshots. This means that the process of
|
||||
creating a snapshot, which is in itself transactional (it happens within
|
||||
a transaction and thus observes some consistent database state), must
|
||||
happen in the same transaction. This is achieved by the master starting
|
||||
a snapshot generating transaction and triggering the process on all
|
||||
workers in the cluster.
|
||||
|
||||
## WAL
|
||||
|
||||
Unlike the snapshot, write-ahead logs contain the information on which
|
||||
transaction made which state change. This makes it possible to include
|
||||
or exclude transactions during the recovery process. What is necessary
|
||||
however is a global consensus on which of the transactions should be
|
||||
recovered and which not, to ensure recovery into a consistent state.
|
||||
|
||||
It would be possible to achieve this with some kind of synchronized
|
||||
recovery process, but it would impose constraints on cluster startup and
|
||||
would not be trivial.
|
||||
|
||||
A simpler alternative is that the consensus is achieved beforehand,
|
||||
while the database (to be recovered) is still operational. What is
|
||||
necessary is to keep track of which transactions are guaranteed to
|
||||
have been flushed to the WAL files on all the workers in the cluster. It
|
||||
makes sense to keep this record on the master, so a mechanism is
|
||||
introduced which periodically pings all the workers, telling them to
|
||||
flush their WALs, and writes some sort of a log indicating that this has
|
||||
been confirmed. The downside of this is a periodic broadcast must be
|
||||
done, and that potentially slightly less data can be recovered in the
|
||||
case of a crash then if using a post-crash consensus. It is however much
|
||||
simpler to implement.
|
||||
@ -1,51 +0,0 @@
|
||||
## Dynamic Graph Partitioning
|
||||
|
||||
Memgraph supports dynamic graph partitioning similar to the Spinner algorithm,
|
||||
mentioned in this paper: [https://arxiv.org/pdf/1404.3861.pdf].
|
||||
|
||||
Dgp is useful because it tries to group `local` date on the same worker, i.e.
|
||||
it tries to keep closely connected data on one worker. It tries to avoid jumps
|
||||
across workers when querying/traversing the distributed graph.
|
||||
|
||||
### Our implementation
|
||||
|
||||
It works independently on each worker but it is always running the migration
|
||||
on only one worker at the same time. It achieves that by sharing a token
|
||||
between workers, and the token ownership is transferred to the next worker
|
||||
when the current worker finishes its migration step.
|
||||
|
||||
The reason that we want workers to work in disjoint time slots is it avoid
|
||||
serialization errors caused by creating/removing edges of vertices during
|
||||
migrations, which might cause an update of some vertex from two or more
|
||||
different transactions.
|
||||
|
||||
### Migrations
|
||||
|
||||
For each vertex and workerid (label in the context of Dgp algorithm) we define
|
||||
a score function. Score function takes into account labels of surrounding
|
||||
endpoints of vertex edges (in/out) and the capacity of the worker with said
|
||||
label. Score function loosely looks like this
|
||||
```
|
||||
locality(v, l) =
|
||||
count endpoints of edges of vertex `v` with label `l` / degree of `v`
|
||||
|
||||
capacity(l) =
|
||||
number of vertices on worker `l` divided by the worker capacity
|
||||
(usually equal to the average number of vertices per worker)
|
||||
|
||||
score(v, l) = locality(v, l) - capacity(l)
|
||||
```
|
||||
We also define two flags alongside ```dynamic_graph_partitioner_enabled```,
|
||||
```dgp_improvement_threshold``` and ```dgp_max_batch_size```.
|
||||
|
||||
These two flags are used during the migration phase.
|
||||
When deciding if we need to migrate some vertex `v` from worker `l1` to worker
|
||||
`l2` we examine the difference in scores, i.e.
|
||||
if score(v, l1) - dgp_improvement_threshold / 100 < score(v, l2) then we
|
||||
migrate the vertex.
|
||||
|
||||
Max batch size flag limits the number of vertices we can transfer in one batch
|
||||
(one migration step).
|
||||
Setting this value to a too large value will probably cause
|
||||
a lot of interference with client queries, and having it a small value
|
||||
will slow down convergence of the algorithm.
|
||||
@ -1,54 +0,0 @@
|
||||
# Memgraph distributed
|
||||
|
||||
This chapter describes some of the concepts used in distributed
|
||||
Memgraph. By "distributed" here we mean the sharding of a single graph
|
||||
onto multiple processing units (servers).
|
||||
|
||||
## Conceptual organization
|
||||
|
||||
There is a single master and multiple workers. The master contains all
|
||||
the global sources of truth (transaction engine,
|
||||
[label|edge-type|property] to name mappings). Also, in the current
|
||||
organization it is the only one that contains a Bolt server (for
|
||||
communication with the end client) and an interpretation engine. Workers
|
||||
contain the data and means of subquery interpretation (query plans
|
||||
recieved from the master) and means of communication with the master and
|
||||
other workers.
|
||||
|
||||
In many query plans the load on the master is much larger then the load
|
||||
on the workers. For that reason it might be beneficial to make the
|
||||
master contain less data (or none at all), and/or having multiple
|
||||
interpretation masters.
|
||||
|
||||
## Logic organization
|
||||
|
||||
Both the distributed and the single node Memgraph use the same codebase.
|
||||
In cases where the behavior in single-node differs from that in
|
||||
distributed, some kind of dynamic behavior change is implemented (either
|
||||
through inheritance or conditional logic).
|
||||
|
||||
### GraphDb
|
||||
|
||||
The `database::GraphDb` is an "umbrella" object for parts of the
|
||||
database such as storage, garbage collection, transaction engine etc.
|
||||
There is a class heirarchy of `GraphDb` implementations, as well as a
|
||||
base interface object. There are subclasses for single-node, master and
|
||||
worker deplotyments. Which implementation is used depends on the
|
||||
configuration processed in the `main` entry point of memgraph.
|
||||
|
||||
The `GraphDb` interface exposes getters to base classes of
|
||||
other similar heirarchies (for example to `tx::Engine`). In that way
|
||||
much of the code that uses those objects (for example query plan
|
||||
interpretation) is agnostic to the type of deployment.
|
||||
|
||||
### RecordAccessors
|
||||
|
||||
The functionality of `RecordAccessors` and it's subclasses is already
|
||||
documented. It's important to note that the same implementation of
|
||||
accessors is used in all deployments, with internal changes of behavior
|
||||
depending on the locality of the graph element (vertex or edge) the
|
||||
accessor represents. For example, if the graph element is local, an
|
||||
update operation on an accessor will make the necessary MVCC ops, update
|
||||
local data, indexes, the write-ahead log etc. However, if the accessor
|
||||
represents a remote graph element, an update will trigger an RPC message
|
||||
to the owner about the update and a change in the local cache.
|
||||
@ -1,103 +0,0 @@
|
||||
# Distributed updates
|
||||
|
||||
Operations that modify the graph state are somewhat more complex in the
|
||||
distributed system, as opposed to a single-node Memgraph deployment. The
|
||||
complexity arises from two factors.
|
||||
|
||||
First, the data being modified is not necessarily owned by the worker
|
||||
performing the modification. This situation is completely valid workers
|
||||
execute parts of the query plan and parts must be executed by the
|
||||
master.
|
||||
|
||||
Second, there are less guarantees regarding multi-threaded access. In
|
||||
single-node Memgraph it was guaranteed that only one transaction will be
|
||||
performing database work in a single transaction. This implied that
|
||||
per-version storage could be thread-unsafe. In distributed Memgraph it
|
||||
is possible that multiple threads could be performing work in the same
|
||||
transaction as a consequence of the query being executed at the same
|
||||
time on multiple workers and those executions interacting with the
|
||||
globally partitioned database state.
|
||||
|
||||
## Deferred state modification
|
||||
|
||||
Making the per-version data storage thread-safe would most likely have a
|
||||
performance impact very undesirable in a transactional database intended
|
||||
for high throughput.
|
||||
|
||||
An alternative is that state modification over unsafe structures is not
|
||||
performed immediately when requested, but postponed until it is safe to
|
||||
do (there is is a guarantee of no concurrent access).
|
||||
|
||||
Since local query plan execution is done the same way on local data as
|
||||
it is in single-node Memgraph, it is not possible to deffer that part of
|
||||
the modification story. What can be deferred are modifications requested
|
||||
by other workers. Since local query plan execution still is
|
||||
single-threaded, this approach is safe.
|
||||
|
||||
At the same time those workers requesting the remote update can update
|
||||
local copies (caches) of the not-owned data since that cache is only
|
||||
being used by the single, local-execution thread.
|
||||
|
||||
### Visibility
|
||||
|
||||
Since updates are deferred the question arises: when do the updates
|
||||
become visible? The above described process offers the following
|
||||
visibility guarantees:
|
||||
- updates done on the local state are visible to the owner
|
||||
- updates done on the local state are NOT visible to anyone else during
|
||||
the same (transaction + command)
|
||||
- updates done on remote state are deferred on the owner and not
|
||||
visible to the owner until applied
|
||||
- updates done on the remote state are applied immediately to the local
|
||||
caches and thus visible locally
|
||||
|
||||
This implies an inconsistent view of the database state. In a concurrent
|
||||
execution of a single query this can hardly be avoided and is accepted
|
||||
as such. It does not change the Cypher query execution semantic in any
|
||||
of the well-defined scenarios. It possibly changes some of the behaviors
|
||||
in which the semantic is not well defined even in single-node execution.
|
||||
|
||||
### Synchronization, update application
|
||||
|
||||
In many queries it is mandatory to observe the latest global graph state
|
||||
(typically when returning it to the client). That means that before that
|
||||
happens all the deferred updates need to be applied, and all the caches
|
||||
to remote data invalidated. Exactly this happens when executing queries
|
||||
that modify the graph state. At some point a global synchronization
|
||||
point is reached. First it is waited that all workers finish the
|
||||
execution of query plan parts performing state modifications. After that
|
||||
all the workers are told to apply the deferred updates they received to
|
||||
their graph state. Since there is no concurrent query plan execution,
|
||||
this is safe. Once that is done all the local caches are cleared and the
|
||||
requested data can be returned to the client.
|
||||
|
||||
### Command advancement
|
||||
|
||||
In complex queries where a read part follows a state modification part
|
||||
the synchronization process after the state modification part is
|
||||
followed by command advancement, like in single-node execution.
|
||||
|
||||
## Creation
|
||||
|
||||
Graph element creation is not deferred. This is practical because the
|
||||
response to a creation is the global ID of the newly created element. At
|
||||
the same time it is safe because no other worker (including the owner)
|
||||
will be using the newly added graph element.
|
||||
|
||||
## Updating
|
||||
|
||||
Updating is deferred, as described. Note that this also means that
|
||||
record locking conflicts are deferred and serialization errors
|
||||
(including lock timeouts) are postponed until the deferred update
|
||||
application phase. In certain scenarios it might be beneficial to force
|
||||
these errors to happen earlier, when the deferred update request is
|
||||
processed.
|
||||
|
||||
## Deletion
|
||||
|
||||
Deletion is also deferred. Deleting an edge implies a modification of
|
||||
it's endpoint vertices, which must be deferred as those data structures
|
||||
are not thread-safe. Deleting a vertex is either with detaching, in
|
||||
which case an arbitrary number of updates are implied in the vertex's
|
||||
neighborhood, or without detaching which relies on checking the current
|
||||
state of the graph which is generally impossible in distributed.
|
||||
@ -1,20 +0,0 @@
|
||||
## Dynamic Graph Partitioner
|
||||
|
||||
Memgraph supports dynamic graph partitioning which dynamically improves
|
||||
performance on badly partitioned dataset over workers. To enable it, the user
|
||||
should use the following flag when firing up the *master* node:
|
||||
|
||||
```plaintext
|
||||
--dynamic_graph_partitioner_enable
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
| Name | Default Value | Description | Range |
|
||||
|------|---------------|-------------|-------|
|
||||
|--dgp_improvement_threshold | 10 | How much better should specific node score
|
||||
be to consider a migration to another worker. This represents the minimal
|
||||
difference between new score that the vertex will have when migrated and the
|
||||
old one such that it's migrated. | Min: 1, Max: 100
|
||||
|--dgp_max_batch_size | 2000 | Maximal amount of vertices which should be
|
||||
migrated in one dynamic graph partitioner step. | Min: 1, Max: MaxInt32 |
|
||||
Loading…
Reference in New Issue
Block a user