From e28d15c27217fc2246dd797bb1daebc721e9bead Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dominik=20Tomic=CC=8Cevic=CC=81?=
<dominik.tomicevic@gmail.com>
Date: Sun, 13 Sep 2015 11:34:17 +0200
Subject: [PATCH] worked on mvcc, compiler and some utils, unfinished
---
cypher/ast/ast.hpp | 5 +
cypher/ast/ast_visitor.hpp | 11 +-
cypher/ast/match.hpp | 22 ++++
cypher/ast/pattern.hpp | 12 +--
cypher/ast/queries.hpp | 22 ++++
cypher/ast/relationship.hpp | 8 +-
cypher/ast/start.hpp | 20 ++++
cypher/ast/tree.hpp | 4 +-
cypher/ast/where.hpp | 20 ++++
cypher/cypher.y | 46 +++++---
cypher/errors.hpp | 26 +++++
cypher/lexer.hpp | 9 +-
cypher/lexical_error.hpp | 15 ---
data_structures/list/lockfree_list.hpp | 108 +++++++++++++++++++
data_structures/list/lockfree_list2.hpp | 71 ++++++++++++
data_structures/skiplist/skiplist.hpp | 4 +-
data_structures/skiplist/skipnode.hpp | 2 +-
data_structures/slstack.hpp | 13 ++-
main.cpp | 2 +-
memory/{memory_engine.hpp => memory.hpp} | 5 +-
mvcc/atom.hpp | 57 ++++++++++
{transaction => mvcc}/commit_log.hpp | 4 +-
{storage/model/utils => mvcc}/minmax.hpp | 9 +-
{storage/model/utils => mvcc}/mvcc.hpp | 27 +++--
mvcc/store.hpp | 85 +++++++++++++++
{transaction => mvcc}/transaction.hpp | 6 +-
{transaction => mvcc}/transaction_engine.hpp | 9 +-
{storage/model/utils => mvcc}/version.hpp | 9 +-
server/uv/uvbuffer.inl | 6 +-
storage/{model => }/edge.hpp | 8 +-
storage/graph.hpp | 33 ++++++
storage/model/graph.hpp | 16 ---
storage/model/properties/properties.hpp | 2 +-
storage/model/properties/property.hpp | 2 +-
storage/model/properties/string.hpp | 2 +-
storage/model/record.hpp | 15 +--
storage/model/root.hpp | 19 ----
storage/storage_engine.hpp | 2 +
storage/{model => }/vertex.hpp | 6 +-
threading/sync/spinlock.hpp | 4 +-
threading/sync/timed_spinlock.hpp | 55 ++++++++++
utils/buffer.hpp | 47 ++++++++
utils/counters/atomic_counter.hpp | 2 +-
utils/memory/atomic_shared_ptr.hpp | 29 +++++
utils/numerics/ceil.hpp | 21 ++++
45 files changed, 764 insertions(+), 136 deletions(-)
create mode 100644 cypher/ast/match.hpp
create mode 100644 cypher/ast/queries.hpp
create mode 100644 cypher/ast/start.hpp
create mode 100644 cypher/ast/where.hpp
create mode 100644 cypher/errors.hpp
delete mode 100644 cypher/lexical_error.hpp
create mode 100644 data_structures/list/lockfree_list.hpp
create mode 100644 data_structures/list/lockfree_list2.hpp
rename memory/{memory_engine.hpp => memory.hpp} (89%)
create mode 100644 mvcc/atom.hpp
rename {transaction => mvcc}/commit_log.hpp (91%)
rename {storage/model/utils => mvcc}/minmax.hpp (84%)
rename {storage/model/utils => mvcc}/mvcc.hpp (91%)
create mode 100644 mvcc/store.hpp
rename {transaction => mvcc}/transaction.hpp (91%)
rename {transaction => mvcc}/transaction_engine.hpp (88%)
rename {storage/model/utils => mvcc}/version.hpp (88%)
rename storage/{model => }/edge.hpp (52%)
create mode 100644 storage/graph.hpp
delete mode 100644 storage/model/graph.hpp
delete mode 100644 storage/model/root.hpp
rename storage/{model => }/vertex.hpp (58%)
create mode 100644 threading/sync/timed_spinlock.hpp
create mode 100644 utils/buffer.hpp
create mode 100644 utils/memory/atomic_shared_ptr.hpp
create mode 100644 utils/numerics/ceil.hpp
diff --git a/cypher/ast/ast.hpp b/cypher/ast/ast.hpp
index 3fc87a0be..1a0cf962c 100644
--- a/cypher/ast/ast.hpp
+++ b/cypher/ast/ast.hpp
@@ -9,5 +9,10 @@
#include "relationship.hpp"
#include "node.hpp"
#include "return.hpp"
+#include "pattern.hpp"
+#include "return.hpp"
+#include "match.hpp"
+#include "queries.hpp"
+#include "start.hpp"
#endif
diff --git a/cypher/ast/ast_visitor.hpp b/cypher/ast/ast_visitor.hpp
index e296dd31a..043b00817 100644
--- a/cypher/ast/ast_visitor.hpp
+++ b/cypher/ast/ast_visitor.hpp
@@ -34,18 +34,27 @@ struct Star;
struct Slash;
struct Rem;
+struct RelationshipSpecs;
struct RelationshipList;
struct Relationship;
struct Node;
struct LabelList;
+struct Pattern;
+
struct ReturnList;
+struct Match;
+struct Where;
+struct ReadQuery;
+
+struct Start;
struct AstVisitor : Visitor<Accessor, Boolean, Float, Identifier, Integer,
String, Property, And, Or, Lt, Gt, Ge, Le, Eq, Ne, Plus, Minus, Star,
Slash, Rem, PropertyList, RelationshipList, Relationship, Node,
- LabelList, ReturnList> {};
+ RelationshipSpecs, LabelList, ReturnList, Pattern, Match, ReadQuery,
+ Start, Where> {};
}
diff --git a/cypher/ast/match.hpp b/cypher/ast/match.hpp
new file mode 100644
index 000000000..b50155ceb
--- /dev/null
+++ b/cypher/ast/match.hpp
@@ -0,0 +1,22 @@
+#ifndef MEMGRAPH_CYPHER_AST_MATCH_HPP
+#define MEMGRAPH_CYPHER_AST_MATCH_HPP
+
+#include "ast_node.hpp"
+#include "pattern.hpp"
+#include "where.hpp"
+
+namespace ast
+{
+
+struct Match : public AstNode<Match>
+{
+ Match(Pattern* pattern, Where* where)
+ : pattern(pattern), where(where) {}
+
+ Pattern* pattern;
+ Where* where;
+};
+
+}
+
+#endif
diff --git a/cypher/ast/pattern.hpp b/cypher/ast/pattern.hpp
index 397d1ca3d..2337abdad 100644
--- a/cypher/ast/pattern.hpp
+++ b/cypher/ast/pattern.hpp
@@ -2,20 +2,18 @@
#define MEMGRAPH_CYPHER_AST_PATTERN_HPP
#include "ast_node.hpp"
+#include "relationship.hpp"
namespace ast
{
-template <class Derived>
-struct Direction : public AstNode<Derived> {};
-
-struct DirectionLeft : public Direction<DirectionLeft> {};
-struct DirectionRight : public Direction<DirectionLeft> {};
-struct Unidirectional : public Direction<DirectionLeft> {};
-
struct Pattern : public AstNode<Pattern>
{
+ Pattern(Node* node, Relationship* relationship, Pattern* next)
+ : node(node), relationship(relationship), next(next) {}
+
Node* node;
+ Relationship* relationship;
Pattern* next;
};
diff --git a/cypher/ast/queries.hpp b/cypher/ast/queries.hpp
new file mode 100644
index 000000000..c1c5a8692
--- /dev/null
+++ b/cypher/ast/queries.hpp
@@ -0,0 +1,22 @@
+#ifndef MEMGRAPH_CYPHER_AST_QUERIES_HPP
+#define MEMGRAPH_CYPHER_AST_QUERIES_HPP
+
+#include "ast_node.hpp"
+#include "match.hpp"
+#include "return.hpp"
+
+namespace ast
+{
+
+struct ReadQuery : public AstNode<ReadQuery>
+{
+ ReadQuery(Match* match, ReturnList* return_list)
+ : match(match), return_list(return_list) {}
+
+ Match* match;
+ ReturnList* return_list;
+};
+
+}
+
+#endif
diff --git a/cypher/ast/relationship.hpp b/cypher/ast/relationship.hpp
index 99fa4d73a..a6d20a1e8 100644
--- a/cypher/ast/relationship.hpp
+++ b/cypher/ast/relationship.hpp
@@ -24,7 +24,13 @@ struct RelationshipSpecs : public AstNode<RelationshipSpecs>
struct Relationship : public AstNode<Relationship>
{
- Relationship()
+ enum Direction { Left, Right, Both };
+
+ Relationship(RelationshipSpecs* specs, Direction direction)
+ : specs(specs), direction(direction) {}
+
+ RelationshipSpecs* specs;
+ Direction direction;
};
}
diff --git a/cypher/ast/start.hpp b/cypher/ast/start.hpp
new file mode 100644
index 000000000..93ba59d05
--- /dev/null
+++ b/cypher/ast/start.hpp
@@ -0,0 +1,20 @@
+#ifndef MEMGRAPH_CYPHER_AST_START_HPP
+#define MEMGRAPH_CYPHER_AST_START_HPP
+
+#include "ast_node.hpp"
+#include "queries.hpp"
+
+namespace ast
+{
+
+struct Start : public AstNode<Start>
+{
+ Start(ReadQuery* read_query)
+ : read_query(read_query) {}
+
+ ReadQuery* read_query;
+};
+
+};
+
+#endif
diff --git a/cypher/ast/tree.hpp b/cypher/ast/tree.hpp
index 89225e0a4..c3db1c216 100644
--- a/cypher/ast/tree.hpp
+++ b/cypher/ast/tree.hpp
@@ -13,7 +13,7 @@ class Ast
public:
Ast() {}
- AstVisitable::uptr root;
+ AstVisitable* root;
void traverse(AstVisitor& visitor)
{
@@ -32,7 +32,7 @@ private:
// basically a gc vector that destroys all ast nodes once this object is
// destroyed. parser generator is written in c and works only with raw
// pointers so this is what makes it leak free after the parser finishes
- // parsing
+ // parsing without using shared pointers
std::vector<AstVisitable::uptr> items;
};
diff --git a/cypher/ast/where.hpp b/cypher/ast/where.hpp
new file mode 100644
index 000000000..2da3e2af9
--- /dev/null
+++ b/cypher/ast/where.hpp
@@ -0,0 +1,20 @@
+#ifndef MEMGRAPH_CYPHER_AST_WHERE_HPP
+#define MEMGRAPH_CYPHER_AST_WHERE_HPP
+
+#include "ast_node.hpp"
+#include "expr.hpp"
+
+namespace ast
+{
+
+struct Where : public AstNode<Where>
+{
+ Where(Expr* expr)
+ : expr(expr) {}
+
+ Expr* expr;
+};
+
+}
+
+#endif
diff --git a/cypher/cypher.y b/cypher/cypher.y
index e61aacef3..ddcdb1bac 100644
--- a/cypher/cypher.y
+++ b/cypher/cypher.y
@@ -13,13 +13,12 @@
%syntax_error
{
- std::cout << "syntax error near '" << TOKEN->value << "'." << std::endl;
- throw "";
+ throw SyntaxError(TOKEN->value);
}
%stack_overflow
{
- std::cout << "parser stack overflow" << std::endl;
+ throw ParserError("Parser stack overflow");
}
%name cypher_parser
@@ -31,6 +30,7 @@
#include <cstdlib>
#include "token.hpp"
+ #include "errors.hpp"
#include "ast/ast.hpp"
#include "ast/tree.hpp"
@@ -46,33 +46,45 @@
%left PLUS MINUS.
%left STAR SLASH REM.
-start ::= read_query.
+start ::= read_query(RQ). {
+ ast->root = ast->create<ast::Start>(RQ);
+}
-read_query ::= match_clause return_clause.
+%type read_query {ast::ReadQuery*}
-match_clause ::= MATCH pattern where_clause.
+read_query(RQ) ::= match_clause(M) return_clause(R). {
+ RQ = ast->create<ast::ReadQuery>(M, R);
+}
+
+%type match_clause {ast::Match*}
+
+match_clause(M) ::= MATCH pattern(P) where_clause(W). {
+ M = ast->create<ast::Match>(P, W);
+}
%type pattern {ast::Pattern*}
// pattern specification
-pattern ::= node rel pattern. {
-
+pattern(P) ::= node(N) rel(R) pattern(NEXT). {
+ P = ast->create<ast::Pattern>(N, R, NEXT);
}
-pattern ::= node. {
-
+pattern(P) ::= node(N). {
+ P = ast->create<ast::Pattern>(N, nullptr, nullptr);
}
-rel ::= MINUS rel_spec MINUS. { // unidirectional
+%type rel {ast::Relationship*}
+rel(R) ::= MINUS rel_spec(S) MINUS. { // unidirectional
+ R = ast->create<ast::Relationship>(S, ast::Relationship::Both);
}
-rel ::= LT MINUS rel_spec MINUS { // left
-
+rel(R) ::= LT MINUS rel_spec(S) MINUS. { // left
+ R = ast->create<ast::Relationship>(S, ast::Relationship::Left);
}
-rel(R) ::= MINUS rel_spec(S) MINUS GT { // right
- R = ast->create<ast::Relationship>(
+rel(R) ::= MINUS rel_spec(S) MINUS GT. { // right
+ R = ast->create<ast::Relationship>(S, ast::Relationship::Right);
}
%type rel_spec {ast::RelationshipSpecs*}
@@ -148,11 +160,11 @@ label_idn(L) ::= . {
L = nullptr;
}
-%type where_clause {ast::Expr*}
+%type where_clause {ast::Where*}
// where clause
where_clause(W) ::= WHERE expr(E). {
- W = E;
+ W = ast->create<ast::Where>(E);
}
where_clause(W) ::= . {
diff --git a/cypher/errors.hpp b/cypher/errors.hpp
new file mode 100644
index 000000000..e5bea3625
--- /dev/null
+++ b/cypher/errors.hpp
@@ -0,0 +1,26 @@
+#ifndef MEMGRAPH_CYPHER_ERRORS_HPP
+#define MEMGRAPH_CYPHER_ERRORS_HPP
+
+#include <stdexcept>
+#include "token.hpp"
+
+class SyntaxError : public std::runtime_error
+{
+public:
+ SyntaxError(const std::string& near)
+ : std::runtime_error("Syntax error near '" + near + "'.") {}
+};
+
+class LexicalError : public std::runtime_error
+{
+public:
+ LexicalError(const Token& token)
+ : std::runtime_error("Unrecognized token '" + token.value + "'.") {}
+};
+
+class ParserError : public std::runtime_error
+{
+ using runtime_error::runtime_error;
+};
+
+#endif
diff --git a/cypher/lexer.hpp b/cypher/lexer.hpp
index 9353407c4..6a0b15226 100644
--- a/cypher/lexer.hpp
+++ b/cypher/lexer.hpp
@@ -3,16 +3,21 @@
#include <cstdint>
+// unfortunatelly, lexertl uses some stuff deprecated in c++11 so we get some
+// warnings during compile time, mainly for the auto_ptr
+// auto_ptr<lexertl::detail::basic_re_token<char, char> > is deprecated
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#include "lexertl/generator.hpp"
#include "lexertl/lookup.hpp"
+#pragma GCC diagnostic pop
-#include "lexical_error.hpp"
+#include "errors.hpp"
#include "token.hpp"
class Lexer
{
public:
-
class Tokenizer
{
public:
diff --git a/cypher/lexical_error.hpp b/cypher/lexical_error.hpp
deleted file mode 100644
index c11ca3a35..000000000
--- a/cypher/lexical_error.hpp
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef MEMGRAPH_CYPHER_LEXER_LEXICAL_ERROR_HPP
-#define MEMGRAPH_CYPHER_LEXER_LEXICAL_ERROR_HPP
-
-#include <stdexcept>
-
-#include "token.hpp"
-
-class LexicalError : public std::runtime_error
-{
-public:
- LexicalError(const Token& token)
- : std::runtime_error("Unrecognized token '" + token.value + "'.") {}
-};
-
-#endif
diff --git a/data_structures/list/lockfree_list.hpp b/data_structures/list/lockfree_list.hpp
new file mode 100644
index 000000000..96185904d
--- /dev/null
+++ b/data_structures/list/lockfree_list.hpp
@@ -0,0 +1,108 @@
+#ifndef MEMGRAPH_DATA_STRUCTURES_LIST_LOCKFREE_LIST_HPP
+#define MEMGRAPH_DATA_STRUCTURES_LIST_LOCKFREE_LIST_HPP
+
+#include <atomic>
+#include <memory>
+#include <unistd.h>
+
+namespace lockfree
+{
+
+template <class T, size_t sleep_time = 250>
+class List
+{
+ struct Node
+ {
+ T item;
+ std::shared_ptr<Node> next;
+ };
+
+public:
+ List() = default;
+
+ // the default destructor is recursive so it could blow the stack if the
+ // list is long enough. the head node is destructed first via a shared_ptr
+ // and then it automatically destructs the second node and the second
+ // destructs the third and so on. keep that in mind
+ ~List() = default;
+
+ List(List&) = delete;
+ void operator=(List&) = delete;
+
+ class iterator
+ {
+ public:
+ iterator(std::shared_ptr<Node> ptr) : ptr(ptr) {}
+
+ T& operator*() { return ptr->item; }
+ T* operator->() { return &ptr->item; }
+
+ iterator& operator++()
+ {
+ if(ptr)
+ ptr = ptr->next;
+
+ return *this;
+ }
+
+ iterator& operator++(int)
+ {
+ operator++();
+ return *this;
+ }
+
+ private:
+ std::shared_ptr<Node> ptr;
+ };
+
+ iterator begin() { return iterator(std::move(head.load())); }
+ iterator end() { return iterator(nullptr); }
+
+ auto find(T item)
+ {
+ auto p = head.load();
+
+ while(p && p->item != item)
+ p = p->next;
+
+ return iterator(std::move(p));
+ }
+
+ iterator push_front(T item)
+ {
+ auto p = std::make_shared<Node>();
+ p->item = item;
+ p->next = std::move(head.load());
+
+ while(!head.compare_exchange_weak(p->next, p))
+ usleep(sleep_time);
+
+ return iterator(p);
+ }
+
+ iterator pop_front()
+ {
+ auto p = head.load();
+ auto q = p;
+
+ while(p && !head.compare_exchange_weak(p, p->next))
+ {
+ q = p;
+ usleep(sleep_time);
+ }
+
+ return iterator(q);
+ }
+
+ // TODO think about how to make this lock free and safe from ABA
+ // this can easily be thread safe if there is ONLY ONE concurrent
+ // remove operation
+ //bool remove(T item);
+
+private:
+ std::atomic<std::shared_ptr<Node>> head { nullptr };
+};
+
+};
+
+#endif
diff --git a/data_structures/list/lockfree_list2.hpp b/data_structures/list/lockfree_list2.hpp
new file mode 100644
index 000000000..a9d762cff
--- /dev/null
+++ b/data_structures/list/lockfree_list2.hpp
@@ -0,0 +1,71 @@
+#ifndef MEMGRAPH_DATA_STRUCTURES_LOCKFREE_LIST2_HPP
+#define MEMGRAPH_DATA_STRUCTURES_LOCKFREE_LIST2_HPP
+
+#include <atomic>
+
+#include "threading/sync/spinlock.hpp"
+#include "threading/sync/lockable.hpp"
+
+namespace lockfree
+{
+
+template <class T>
+class List2 : Lockable<SpinLock>
+{
+ struct Node { T value; std::atomic<Node*> next; };
+
+ class iterator
+ {
+ iterator(Node* node) : node(node) {}
+
+ T& operator*() const { return *node; }
+ T* operator->() const { return node; }
+
+ bool end() const
+ {
+ return node == nullptr;
+ }
+
+ iterator& operator++()
+ {
+ node = node->next.load(std::memory_order_relaxed);
+ return *this;
+ }
+
+ iterator& operator++(int)
+ {
+ return operator++();
+ }
+
+ protected:
+ Node* node;
+ };
+
+public:
+
+ ~List2()
+ {
+ Node* next, node = head.load(std::memory_order_relaxed);
+
+ for(; node != nullptr; node = next)
+ {
+ next = node->next;
+ delete node;
+ }
+ }
+
+ void insert(T value)
+ {
+ auto guard = acquire();
+ head.store(new Node { value, head });
+ }
+
+ iterator begin() { return iterator(head.load()); }
+
+private:
+ std::atomic<Node*> head;
+};
+
+}
+
+#endif
diff --git a/data_structures/skiplist/skiplist.hpp b/data_structures/skiplist/skiplist.hpp
index 6916865fc..d0d380cef 100644
--- a/data_structures/skiplist/skiplist.hpp
+++ b/data_structures/skiplist/skiplist.hpp
@@ -84,11 +84,9 @@ public:
lfound = level;
preds[level] = pred;
- succs[level] = node; // TODO what's FB doing here?
+ succs[level] = node;
}
- //std::cout << "lfound = " << lfound << std::endl;
-
return lfound;
}
diff --git a/data_structures/skiplist/skipnode.hpp b/data_structures/skiplist/skipnode.hpp
index e03d76f7f..1aeb45fce 100644
--- a/data_structures/skiplist/skipnode.hpp
+++ b/data_structures/skiplist/skipnode.hpp
@@ -5,7 +5,7 @@
#include <atomic>
#include <mutex>
-#include "sync/spinlock.hpp"
+#include "threading/sync/spinlock.hpp"
// concurrent skiplist node based on the implementation described in
// "A Provably Correct Scalable Concurrent Skip List"
diff --git a/data_structures/slstack.hpp b/data_structures/slstack.hpp
index 7f2b19d65..dbde180a4 100644
--- a/data_structures/slstack.hpp
+++ b/data_structures/slstack.hpp
@@ -3,30 +3,29 @@
#include <stack>
-#include "utils/sync/spinlock.hpp"
+#include "threading/sync/spinlock.hpp"
+#include "threading/sync/lockable.hpp"
template <class T>
-class SpinLockStack
+class SpinLockStack : Lockable<SpinLock>
{
public:
T pop()
{
- lock.acquire();
+ auto guard = acquire();
T elem = stack.top();
stack.pop();
- lock.release();
-
return elem;
}
void push(const T& elem)
{
- lock.acquire();
+ auto guard = acquire();
+
stack.push(elem);
- lock.release();
}
private:
diff --git a/main.cpp b/main.cpp
index 131548399..355567605 100644
--- a/main.cpp
+++ b/main.cpp
@@ -1,7 +1,7 @@
#include <iostream>
#include <thread>
-#include "sync/spinlock.hpp"
+#include "threading/sync/spinlock.hpp"
#include "transaction/transaction_engine.hpp"
#include "memory/memory_engine.hpp"
diff --git a/memory/memory_engine.hpp b/memory/memory.hpp
similarity index 89%
rename from memory/memory_engine.hpp
rename to memory/memory.hpp
index 7735f726b..7d379d70a 100644
--- a/memory/memory_engine.hpp
+++ b/memory/memory.hpp
@@ -1,10 +1,9 @@
-#ifndef MEMGRAPH_MEMORY_MEMORY_ENGINE_HPP
-#define MEMGRAPH_MEMORY_MEMORY_ENGINE_HPP
+#ifndef MEMGRAPH_MEMORY_MEMORY_HPP
+#define MEMGRAPH_MEMORY_MEMORY_HPP
#include <atomic>
#include <mutex>
-#include "transaction/transaction.hpp"
#include "storage/model/record.hpp"
#include "storage/model/vertex.hpp"
#include "storage/model/edge.hpp"
diff --git a/mvcc/atom.hpp b/mvcc/atom.hpp
new file mode 100644
index 000000000..ab3e8570f
--- /dev/null
+++ b/mvcc/atom.hpp
@@ -0,0 +1,57 @@
+#ifndef MEMGRAPH_MVCC_ATOM_HPP
+#define MEMGRAPH_MVCC_ATOM_HPP
+
+#include "threading/sync/lockable.hpp"
+
+#include "transaction.hpp"
+#include "version.hpp"
+
+namespace mvcc
+{
+
+template <class T>
+class Atom : public Version<T>,
+ public Lockable<>
+{
+public:
+ Atom(uint64_t id, T* first)
+ : Version<T>(first), id(id)
+ {
+ // it's illegal that the first version is nullptr. there should be at
+ // least one version of a record
+ assert(first != nullptr);
+ }
+
+ T* first()
+ {
+ return this->newer();
+ }
+
+ // inspects the record change history and returns the record version visible
+ // to the current transaction if it exists, otherwise it returns nullptr
+ T* latest_visible(const Transaction& t)
+ {
+ return first()->latest_visible(t);
+ }
+
+ // inspects the record change history and returns the newest available
+ // version from all transactions
+ T& newest_available()
+ {
+ T* record = this->newer(), newer = this->newer();
+
+ while(newer != nullptr)
+ record = newer, newer = record->newer();
+
+ return record;
+ }
+
+ // every record has a unique id. 2^64 = 1.8 x 10^19. that should be enough
+ // for a looong time :) but keep in mind that some vacuuming would be nice
+ // to reuse indices for deleted nodes.
+ uint64_t id;
+};
+
+}
+
+#endif
diff --git a/transaction/commit_log.hpp b/mvcc/commit_log.hpp
similarity index 91%
rename from transaction/commit_log.hpp
rename to mvcc/commit_log.hpp
index 7a235e876..1a6a2bc11 100644
--- a/transaction/commit_log.hpp
+++ b/mvcc/commit_log.hpp
@@ -5,8 +5,8 @@
#include <boost/dynamic_bitset/dynamic_bitset.hpp>
-#include "sync/spinlock.hpp"
-#include "sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
+#include "threading/sync/lockable.hpp"
// optimize allocation performance by preallocating chunks in like 2MB or so
// optimize memory by purging old transactions after vacuuming or something
diff --git a/storage/model/utils/minmax.hpp b/mvcc/minmax.hpp
similarity index 84%
rename from storage/model/utils/minmax.hpp
rename to mvcc/minmax.hpp
index bca402a4d..29bf76524 100644
--- a/storage/model/utils/minmax.hpp
+++ b/mvcc/minmax.hpp
@@ -1,8 +1,11 @@
-#ifndef MEMGRAPH_STORAGE_MODEL_UTILS_MINMAX_HPP
-#define MEMGRAPH_STORAGE_MODEL_UTILS_MINMAX_HPP
+#ifndef MEMGRAPH_MVCC_MINMAX_HPP
+#define MEMGRAPH_MVCC_MINMAX_HPP
#include <atomic>
+namespace mvcc
+{
+
template <class T>
class MinMax
{
@@ -35,4 +38,6 @@ private:
std::atomic<T> min_, max_;
};
+}
+
#endif
diff --git a/storage/model/utils/mvcc.hpp b/mvcc/mvcc.hpp
similarity index 91%
rename from storage/model/utils/mvcc.hpp
rename to mvcc/mvcc.hpp
index ce1005e77..9d80cb86b 100644
--- a/storage/model/utils/mvcc.hpp
+++ b/mvcc/mvcc.hpp
@@ -3,15 +3,16 @@
#include <atomic>
-#include "transaction/transaction.hpp"
-#include "storage/model/utils/minmax.hpp"
-#include "storage/model/utils/version.hpp"
-
-#include "transaction/commit_log.hpp"
+#include "transaction.hpp"
+#include "minmax.hpp"
+#include "version.hpp"
// the mvcc implementation used here is very much like postgresql's
// more info: https://momjian.us/main/writings/pgsql/mvcc.pdf
+namespace mvcc
+{
+
template <class T>
class Mvcc : public Version<T>
{
@@ -62,7 +63,7 @@ public:
// inspects the record change history and returns the record version visible
// to the current transaction if it exists, otherwise it returns nullptr
- T* latest(const Transaction& t)
+ T* latest_visible(const Transaction& t)
{
T* record = this, newer = this->newer();
@@ -75,6 +76,18 @@ public:
return record;
}
+ void mark_created(const Transaction& t)
+ {
+ tx.min(t.id);
+ cmd.min(t.cid);
+ }
+
+ void mark_deleted(const Transaction& t)
+ {
+ tx.max(t.id);
+ cmd.max(t.cid);
+ }
+
protected:
// known committed and known aborted for both xmax and xmin
// this hints are used to quickly check the commit/abort status of the
@@ -84,4 +97,6 @@ protected:
std::atomic<uint8_t> hints;
};
+}
+
#endif
diff --git a/mvcc/store.hpp b/mvcc/store.hpp
new file mode 100644
index 000000000..8b72d957c
--- /dev/null
+++ b/mvcc/store.hpp
@@ -0,0 +1,85 @@
+#ifndef MEMGRAPH_STORAGE_MVCC_STORE_HPP
+#define MEMGRAPH_STORAGE_MVCC_STORE_HPP
+
+#include <stdexcept>
+
+#include "mvcc/transaction.hpp"
+#include "mvcc/atom.hpp"
+
+#include "data_structures/list/lockfree_list.hpp"
+#include "utils/counters/atomic_counter.hpp"
+
+// some interesting concepts described there, keep in mind for the future
+// Serializable Isolation for Snapshot Databases, J. Cahill, et al.
+
+namespace mvcc
+{
+
+class MvccError : public std::runtime_error
+{
+public:
+ using runtime_error::runtime_error;
+};
+
+template <class T>
+class MvccStore
+{
+ using list_t = lockfree::List<Atom<T>>;
+
+public:
+ using iterator = typename list_t::iterator;
+
+ MvccStore() : counter(0) {}
+
+ iterator insert(const Transaction& t)
+ {
+ auto record = new T();
+
+ record->mark_created(t);
+
+ return data.push_front(Atom<T>(counter.next(), record));
+ }
+
+ T* update(Atom<T>& atom, T& record, const Transaction& t)
+ {
+ auto guard = atom.acquire();
+
+ // if xmax is not zero, that means there is a newer version of this
+ // record or it has been deleted. we cannot do anything here
+ if(record.tx.max())
+ throw MvccError("can't serialize due to concurrent operation(s)");
+
+ // make a new version
+ auto updated = new T();
+ *updated = *record;
+
+ // mark the new version as created
+ updated->mark_created(t);
+
+ // mark the current version as deleted
+ record.mark_deleted(t);
+ record.newer(updated);
+
+ return updated;
+ }
+
+ void remove(Atom<T>& atom, T& record, const Transaction& t)
+ {
+ auto guard = atom.acquire();
+
+ // if xmax is not zero, that means there is a newer version of this
+ // record or it has been deleted. we cannot do anything here
+ if(record.tx.max())
+ throw MvccError("can't serialize due to concurrent operation(s)");
+
+ record.mark_deleted(t);
+ }
+
+private:
+ AtomicCounter<uint64_t> counter;
+ lockfree::List<Atom<T>> data;
+};
+
+}
+
+#endif
diff --git a/transaction/transaction.hpp b/mvcc/transaction.hpp
similarity index 91%
rename from transaction/transaction.hpp
rename to mvcc/transaction.hpp
index 24d814ee4..80a38748c 100644
--- a/transaction/transaction.hpp
+++ b/mvcc/transaction.hpp
@@ -1,11 +1,11 @@
-#ifndef MEMGRAPH_TRANSACTION_TRANSACTION_HPP
-#define MEMGRAPH_TRANSACTION_TRANSACTION_HPP
+#ifndef MEMGRAPH_MVCC_TRANSACTION_HPP
+#define MEMGRAPH_MVCC_TRANSACTION_HPP
#include <cstdlib>
#include <cstdint>
#include <vector>
-#include "transaction/commit_log.hpp"
+#include "commit_log.hpp"
struct Transaction
{
diff --git a/transaction/transaction_engine.hpp b/mvcc/transaction_engine.hpp
similarity index 88%
rename from transaction/transaction_engine.hpp
rename to mvcc/transaction_engine.hpp
index caf9a57ba..d6ce74a91 100644
--- a/transaction/transaction_engine.hpp
+++ b/mvcc/transaction_engine.hpp
@@ -1,5 +1,5 @@
-#ifndef MEMGRAPH_TRANSACTION_TRANSACTIONENGINE_HPP
-#define MEMGRAPH_TRANSACTION_TRANSACTIONENGINE_HPP
+#ifndef MEMGRAPH_MVCC_TRANSACTIONENGINE_HPP
+#define MEMGRAPH_MVCC_TRANSACTIONENGINE_HPP
#include <cstdlib>
#include <atomic>
@@ -12,8 +12,8 @@
#include "transaction.hpp"
#include "utils/counters/simple_counter.hpp"
-#include "sync/spinlock.hpp"
-#include "sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
+#include "threading/sync/lockable.hpp"
class TransactionEngine : Lockable<SpinLock>
{
@@ -47,7 +47,6 @@ public:
finalize(t);
}
- // id of the last finished transaction
uint64_t last_known_active()
{
auto guard = this->acquire();
diff --git a/storage/model/utils/version.hpp b/mvcc/version.hpp
similarity index 88%
rename from storage/model/utils/version.hpp
rename to mvcc/version.hpp
index 36af46039..bbc99a1be 100644
--- a/storage/model/utils/version.hpp
+++ b/mvcc/version.hpp
@@ -1,8 +1,11 @@
-#ifndef MEMGRAPH_STORAGE_MODEL_UTILS_VERSION_HPP
-#define MEMGRAPH_STORAGE_MODEL_UTILS_VERSION_HPP
+#ifndef MEMGRAPH_MVCC_VERSION_HPP
+#define MEMGRAPH_MVCC_VERSION_HPP
#include <atomic>
+namespace mvcc
+{
+
template <class T>
class Version
{
@@ -31,4 +34,6 @@ private:
std::atomic<T*> versions;
};
+}
+
#endif
diff --git a/server/uv/uvbuffer.inl b/server/uv/uvbuffer.inl
index 95afc3082..2fea17b53 100644
--- a/server/uv/uvbuffer.inl
+++ b/server/uv/uvbuffer.inl
@@ -70,7 +70,11 @@ UvBuffer& UvBuffer::append(const char* data, size_t n)
buffer.base = static_cast<char*>(realloc(buffer.base, new_size));
}
- std::memcpy(buffer.base + size(), data, n);
+ auto ptr = buffer.base + size();
+
+ for(size_t i = 0; i < n; ++i, ++ptr, ++data)
+ *ptr = *data;
+
buffer.len = new_size;
return *this;
diff --git a/storage/model/edge.hpp b/storage/edge.hpp
similarity index 52%
rename from storage/model/edge.hpp
rename to storage/edge.hpp
index 207fceb9d..535be028e 100644
--- a/storage/model/edge.hpp
+++ b/storage/edge.hpp
@@ -1,14 +1,12 @@
-#ifndef MEMGRAPH_DATA_MODEL_EDGE_HPP
-#define MEMGRAPH_DATA_MODEL_EDGE_HPP
+#ifndef MEMGRAPH_STORAGE_EDGE_HPP
+#define MEMGRAPH_STORAGE_EDGE_HPP
#include <vector>
-#include "record.hpp"
+#include "model/record.hpp"
struct Vertex;
-struct
-
struct Edge : public Record<Edge>
{
Vertex* from;
diff --git a/storage/graph.hpp b/storage/graph.hpp
new file mode 100644
index 000000000..c0d15acf6
--- /dev/null
+++ b/storage/graph.hpp
@@ -0,0 +1,33 @@
+#ifndef MEMGRAPH_STORAGE_GRAPH_HPP
+#define MEMGRAPH_STORAGE_GRAPH_HPP
+
+#include <list>
+
+#include "mvcc/atom.hpp"
+#include "mvcc/store.hpp"
+
+#include "vertex.hpp"
+#include "edge.hpp"
+
+using VertexStore = mvcc::MvccStore<Vertex>;
+using EdgeStore = mvcc::MvccStore<Edge>;
+
+class Graph
+{
+public:
+ Graph() {}
+
+ EdgeStore::iterator connect(Vertex a, Vertex b, const Transaction& t)
+ {
+ auto it = edges.insert(t);
+
+ it->
+
+ return it;
+ }
+
+ VertexStore vertices;
+ EdgeStore edges;
+};
+
+#endif
diff --git a/storage/model/graph.hpp b/storage/model/graph.hpp
deleted file mode 100644
index c6ab98ce8..000000000
--- a/storage/model/graph.hpp
+++ /dev/null
@@ -1,16 +0,0 @@
-#ifndef MEMGRAPH_DATA_MODEL_GRAPH_HPP
-#define MEMGRAPH_DATA_MODEL_GRAPH_HPP
-
-#include <list>
-
-#include "vertex.hpp"
-#include "edge.hpp"
-
-class Graph
-{
-
-private:
- std::list<
-};
-
-#endif
diff --git a/storage/model/properties/properties.hpp b/storage/model/properties/properties.hpp
index 28dfa76ec..1f47178bf 100644
--- a/storage/model/properties/properties.hpp
+++ b/storage/model/properties/properties.hpp
@@ -5,7 +5,7 @@
#include "property.hpp"
-namespace props
+namespace model
{
class Properties
diff --git a/storage/model/properties/property.hpp b/storage/model/properties/property.hpp
index c01d2e282..1da19b419 100644
--- a/storage/model/properties/property.hpp
+++ b/storage/model/properties/property.hpp
@@ -4,7 +4,7 @@
#include <memory>
#include <string>
-namespace props
+namespace model
{
class Property
diff --git a/storage/model/properties/string.hpp b/storage/model/properties/string.hpp
index 91787f254..49bb82d50 100644
--- a/storage/model/properties/string.hpp
+++ b/storage/model/properties/string.hpp
@@ -3,7 +3,7 @@
#include "property.hpp"
-namespace props
+namespace model
{
class String : public Value<std::string>
diff --git a/storage/model/record.hpp b/storage/model/record.hpp
index cbf6e43a1..0d4f42d27 100644
--- a/storage/model/record.hpp
+++ b/storage/model/record.hpp
@@ -2,24 +2,27 @@
#define MEMGRAPH_STORAGE_RECORD_HPP
#include <mutex>
+#include <set>
#include "utils/crtp.hpp"
-#include "sync/spinlock.hpp"
-#include "sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
-#include "storage/model/utils/mvcc.hpp"
+#include "mvcc/mvcc.hpp"
#include "properties/properties.hpp"
template <class Derived>
class Record
: public Crtp<Derived>,
- public Mvcc<Derived>,
- Lockable<SpinLock>
+ public mvcc::Mvcc<Derived>
{
public:
- Properties props;
+ // a record contains a key value map containing data
+ model::Properties properties;
+
+ // each record can have one or more distinct labels.
+ std::set<uint16_t> labels;
};
#endif
diff --git a/storage/model/root.hpp b/storage/model/root.hpp
deleted file mode 100644
index d52023026..000000000
--- a/storage/model/root.hpp
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef MEMGRAPH_STORAGE_MODEL_ROOT_HPP
-#define MEMGRAPH_STORAGE_MODEL_ROOT_HPP
-
-#include "utils/version.hpp"
-
-template <class T>
-class Root : public Version<T>
-{
-public:
- Root(uint64_t id, T* first)
- : Version<T>(first), id(id) {}
-
- // every record has a unique id. 2^64 = 1.8 x 10^19. that should be enough
- // for a looong time :) but keep in mind that some vacuuming would be nice
- // to reuse indices for deleted nodes.
- uint64_t id;
-};
-
-#endif
diff --git a/storage/storage_engine.hpp b/storage/storage_engine.hpp
index 324646869..61d536afa 100644
--- a/storage/storage_engine.hpp
+++ b/storage/storage_engine.hpp
@@ -9,6 +9,8 @@
#include "memory/memory_engine.hpp"
#include "utils/counters/atomic_counter.hpp"
+#include "storage/model/root.hpp"
+
class StorageEngine
{
public:
diff --git a/storage/model/vertex.hpp b/storage/vertex.hpp
similarity index 58%
rename from storage/model/vertex.hpp
rename to storage/vertex.hpp
index 97bfc9aa8..13e0da874 100644
--- a/storage/model/vertex.hpp
+++ b/storage/vertex.hpp
@@ -1,9 +1,9 @@
-#ifndef MEMGRAPH_STORAGE_MODEL_VERTEX_HPP
-#define MEMGRAPH_STORAGE_MODEL_VERTEX_HPP
+#ifndef MEMGRAPH_STORAGE_VERTEX_HPP
+#define MEMGRAPH_STORAGE_VERTEX_HPP
#include <vector>
-#include "record.hpp"
+#include "model/record.hpp"
#include "edge.hpp"
struct Vertex : public Record<Vertex>
diff --git a/threading/sync/spinlock.hpp b/threading/sync/spinlock.hpp
index bd03b4f07..af218462b 100644
--- a/threading/sync/spinlock.hpp
+++ b/threading/sync/spinlock.hpp
@@ -1,5 +1,5 @@
-#ifndef MEMGRAPH_UTILS_SYNC_SPINLOCK_HPP
-#define MEMGRAPH_UTILS_SYNC_SPINLOCK_HPP
+#ifndef MEMGRAPH_THREADING_SYNC_SPINLOCK_HPP
+#define MEMGRAPH_THREADING_SYNC_SPINLOCK_HPP
#include <atomic>
#include <unistd.h>
diff --git a/threading/sync/timed_spinlock.hpp b/threading/sync/timed_spinlock.hpp
new file mode 100644
index 000000000..5758843b6
--- /dev/null
+++ b/threading/sync/timed_spinlock.hpp
@@ -0,0 +1,55 @@
+#ifndef MEMGRAPH_THREADING_SYNC_TIMED_SPINLOCK_HPP
+#define MEMGRAPH_THREADING_SYNC_TIMED_SPINLOCK_HPP
+
+#include <atomic>
+#include <chrono>
+#include <stdexcept>
+
+#include <unistd.h>
+
+class LockExpiredError : public std::runtime_error
+{
+ using runtime_error::runtime_error;
+};
+
+class TimedSpinLock
+{
+public:
+ TimedSpinLock(std::chrono::seconds expiration)
+ : expiration(expiration) {}
+
+ TimedSpinLock(std::chrono::milliseconds expiration)
+ : expiration(expiration) {}
+
+ void lock()
+ {
+ using clock = std::chrono::high_resolution_clock;
+
+ auto start = clock::now();
+
+ while(lock_flag.test_and_set(std::memory_order_acquire))
+ {
+ // how long have we been locked? if we exceeded the expiration
+ // time, throw an exception and stop being blocked because this
+ // might be a deadlock!
+
+ if(clock::now() - start > expiration)
+ throw LockExpiredError("This lock has expired");
+
+ usleep(250);
+ }
+ }
+
+ void unlock()
+ {
+ lock_flag.clear(std::memory_order_release);
+ }
+
+private:
+ std::chrono::milliseconds expiration;
+
+ // guaranteed by standard to be lock free!
+ std::atomic_flag lock_flag = ATOMIC_FLAG_INIT;
+};
+
+#endif
diff --git a/utils/buffer.hpp b/utils/buffer.hpp
new file mode 100644
index 000000000..584328100
--- /dev/null
+++ b/utils/buffer.hpp
@@ -0,0 +1,47 @@
+#ifndef MEMGRAPH_UTILS_STRING_BUFFER_HPP
+#define MEMGRAPH_UTILS_STRING_BUFFER_HPP
+
+#include <string>
+
+#include "numerics/ceil.hpp"
+
+class Buffer
+{
+public:
+ Buffer(size_t capacity, size_t chunk_size)
+ : capacity(capacity), chunk_size(chunk_size) {}
+
+ Buffer& append(const std::string& string)
+ {
+ return this->append(string.c_str(), string.size());
+ }
+
+ Buffer& append(const char* string, size_t n)
+ {
+ auto new_size = size() + n;
+
+ if(capacity < new_size)
+ {
+ capacity = new_size;
+ data = static_cast<char*>(realloc(data, new_size));
+ }
+
+ size = new_size;
+ }
+
+ Buffer& operator<<(const std::string& string)
+ {
+
+ }
+
+ size_t size() const
+ {
+ return str.size();
+ }
+
+private:
+ size_t size_, capacity, chunk_size;
+ char* data;
+};
+
+#endif
diff --git a/utils/counters/atomic_counter.hpp b/utils/counters/atomic_counter.hpp
index 4fbba83e5..b4671ac80 100644
--- a/utils/counters/atomic_counter.hpp
+++ b/utils/counters/atomic_counter.hpp
@@ -11,7 +11,7 @@ public:
T next()
{
- return counter.fetch_add(1, std::memory_order_relaxed);
+ return counter.fetch_add(1);
}
private:
diff --git a/utils/memory/atomic_shared_ptr.hpp b/utils/memory/atomic_shared_ptr.hpp
new file mode 100644
index 000000000..441029e6f
--- /dev/null
+++ b/utils/memory/atomic_shared_ptr.hpp
@@ -0,0 +1,29 @@
+#ifndef MEMGRAPH_UTILS_MEMORY_ATOMIC_SHARED_PTR_HPP
+#define MEMGRAPH_UTILS_MEMORY_ATOMIC_SHARED_PTR_HPP
+
+#include <atomic>
+#include <memory>
+
+template<class T>
+class atomic_shared_ptr final
+{
+public:
+ atomic_shared_ptr(std::shared_ptr<T>&& ptr)
+ : ptr(ptr) {}
+
+ std::shared_ptr<T> load()
+ {
+ return std::move(std::atomic_load(&ptr));
+ }
+
+ bool compare_exchange_weak(std::shared_ptr<T>& expected,
+ std::shared_ptr<T> desired)
+ {
+ return atomic_compare_exchange_weak(&ptr, &expected, desired);
+ }
+
+private:
+ std::shared_ptr<T> ptr;
+};
+
+#endif
diff --git a/utils/numerics/ceil.hpp b/utils/numerics/ceil.hpp
new file mode 100644
index 000000000..959b661e1
--- /dev/null
+++ b/utils/numerics/ceil.hpp
@@ -0,0 +1,21 @@
+#ifndef MEMGRAPH_UTILS_NUMERICS_COMMON_HPP
+#define MEMGRAPH_UTILS_NUMERICS_COMMON_HPP
+
+#include <type_traits>
+
+namespace num
+{
+
+template <class T,
+ typename std::enable_if<std::is_integral<T>::value>::type* = nullptr>
+T iceil(T x, T y)
+{
+ // this may seem inefficient, but on x86_64, when you already perform
+ // division (x / y) the remainder is already computed and therefore x % y
+ // is basically free!
+ return x / y + (x % y != 0);
+}
+
+}
+
+#endif